Зайти в сессию с requests и продолжить парсинг уже с Selenium

import requests
from lxml import html
import urllib
from bs4 import BeautifulSoup
import io
 
USERNAME = "<USERNAME>"
PASSWORD = "<PASSWORD>"
 
LOGIN_URL = "https://*******.com/site/login"
IR_URL ="https:*************com"
BASE_URL = "https://*************.com/klass?page="
 
parsed_pages=[]
 
def main():
    s = requests.session()
 
    # Get login csrf token
    r = s.get(LOGIN_URL)
    tree = html.fromstring(r.text)
    authenticity_token = list(set(tree.xpath("//input[@name='_csrf-markup']/@value")))[0]
 
    # Create payload
    payload = {
        "LoginForm[name]": "********", 
        "LoginForm[password]": "******", 
        "_csrf-markup": authenticity_token
    }
 
    # Perform login
    r = s.post(LOGIN_URL, data = payload, headers = dict(referer = LOGIN_URL))
    
    print(r.status_code)
    
    # Scrape url
 
    i = list(range(1,2))
 
    for item in i:
        URL = BASE_URL + str(item) + "&per-page=50"
        r = s.get(URL, headers = dict(referer = URL)).content
        soup = BeautifulSoup(r, 'lxml')
 
        table = soup.find("table", attrs={'table table-bordered'})
 
        imgs= table.findAll('img')
        img_links = []
        path = r"C:\*********\class"
 
        for img in imgs:
            print("page:" + str(item), "image:" + img.get('src'))
            img_links.append(img.get('src'))
 
        for img_link in img_links:
            f = io.open('parsed_data.html', 'a', encoding='utf8')
            #write to html file
            f.write("<a href='"+ IR_URL + str(img_link)+"'></a>")
            f.close()
            #save to folder
            
            urllib.request.urlretrieve(IR_URL + str(img_link), path + str(img_link)[str(img_link).rfind("/"):])
 
if __name__ == '__main__':
    main()

login_link = "https://*********.com/site/login"
    link = "https://***********.com/validate?page="
 
    options = webdriver.FirefoxOptions()
    options.add_argument("--headless")
    driver = Firefox(options = options)
 
    try:
        response = driver.request('POST', login_link, data={
        #"_csrf-markup": "erEQFG7wJ6l9K9AHhbg_mLHT1aASSO_XzoY9DPPJgSogxChmJ7l1-h5Dvmb2_hLP3J74ynE62YGayV5LuKfHaA==",
        "LoginForm[name]": "**********",
        "LoginForm[password]": "*********",
        "LoginForm[rememberMe]": "0",
        "login-button": ""
    })
    except TimeoutException as e:
        pass
    time.sleep(2)
    print(response)

driver.get (login_link)
    driver.find_element_by_id("loginform-name").send_keys("*****")
    driver.find_element_by_id ("loginform-password").send_keys("******")
    driver.find_element_by_name("_csrf-markup").setVal
    ('value', "L2b-HHE79HmWXtsTJlK5i7k8Pr9_oAu-EWDqj0lkDcMcNr1zGhaATKMNgmNkCIra7lNu_gWQPtZ-K5rOKyJGpA==")
    driver.find_element_by_name("login-button").click()
 
    driver.get(link)
    time.sleep(5)
    htmlSource = driver.page_source
    print(htmlSource)

    driver.get (login_link)
    driver.find_element_by_id("loginform-name").send_keys("elina.duchinskaya")
    driver.find_element_by_id ("loginform-password").send_keys("8KGGpg14")
    
???? driver.find_element_by_name("_csrf-markup").send_keys
    ('value', "L2b-HHE79HmWXtsTJlK5i7k8Pr9_oAu-EWDqj0lkDcMcNr1zGhaATKMNgmNkCIra7lNu_gWQPtZ-K5rOKyJGpA==")???
 
    driver.find_element_by_name("login-button").click()

import requests
from lxml import html
import urllib
from bs4 import BeautifulSoup
import io
from selenium import webdriver
import time
from seleniumrequests import Firefox
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
 
 
USERNAME = "<USERNAME>"
PASSWORD = "<PASSWORD>"
 
#driver = webdriver.PhantomJS()
options = webdriver.FirefoxOptions()
options.add_argument('--headless')
driver = webdriver.Firefox(options=options)
 
 
login_link = "https://*****************.com/site/login"
IR_link ="https://************.com"
base_link = "https://***************.com/validate?klass_id=5e7dc0691c007-3006&lots_id&size_id&confirm=&page=1&proposal=false&limit=0&klass_search="
 
def main():
   
    driver.get (login_link)
    try:
        element_present = EC.presence_of_element_located((By.NAME, 'login-button'))
        WebDriverWait(driver, 10).until(element_present)
    except TimeoutException:
        print("Timed out waiting for page to load")
 
    driver.find_element_by_id("loginform-name").send_keys("*************")
    driver.find_element_by_id ("loginform-password").send_keys("*******")
    
    driver.find_element_by_name("login-button").click()
 
    print(driver.session_id)
    #WebDriverWait(browser, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "section#header header navbar navbar-default navbar-fixed-top[href='//validate']"))).click()
    #WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "/html/body/div[3]/div[2]/div/div[1]/ul/li[16]/a"))).click()
   
    driver.get (base_link)
    WebDriverWait(driver, 50) #.until(element_present)
    print(driver.page_source)
    html_source = driver.page_source
    data = html_source.encode('utf-8')
    print(data)
 
    driver.quit()
 
 
if __name__ == '__main__':
    main()

driver.find_element_by_xpath("/html/body/div[3]/div/div[2]/form/div[4]/button").click()

>>> url='https://iqtools-grocery-prod.intrtl.com/site/login'
>>> from selenium import webdriver
>>> d = webdriver.Chrome()
>>> d.get(url)
>>> user = '*****'
>>> password='*****'
>>> d.find_element_by_id('loginform-name').send_keys(user)
>>> d.find_element_by_id('loginform-password').send_keys(password)
>>> d.find_element_by_name('login-button').click()

    # отправляем первичный запрос на главную страницу, чтобы получить сессионные куки
    session = requests.Session()
    # шаг первый
    resp = session.get(host,headers=headers)
    html = resp.text
    root = lxml.html.fromstring(html) #  получаем разобранное дерево элементов
    title = root.xpath('//title')[0].text
    
    print(title)
    time.sleep(2)
    # шаг второй
    resp = session.get(login_url,headers=headers)
    html = resp.text
    root = lxml.html.fromstring(html) #  получаем разобранное дерево элементов
    title = root.xpath('//title')[0].text # находим заголовок; по идее он есть практически у всех страничек
    print(title)  # смотрим заголовок страницы
    # проверка что это нужная нам страничка логина; можно не делать и нужно понимать, что заголовок странички логина  может быть другим
    if 'Login' in title:    
        form = root.forms[0]  # берем самую первую форму - то, что брать нужно именно ее - нужно убеждаться эмпирическим путем, заглянув в исходный код страницы, так как нужная форма может и не быть первой
        print(form.form_values())
        # заполняем поля; скрытые поля будут отосланы автоматически
        form.fields['username'] = login      # имена полей узнаем из исходного кода страницы; у вас они могут быть дргуими
        form.fields['password'] = password
        time.sleep(2)
        # шаг третий
        # в form.action  должен быть относительный url, иначе host не нужен.
        resp = session.post("{}{}".format(host,form.action), data=form.form_values())
        ... дальше проверяем что это страничка на которую нас должны перенаправить после удачного логина

def main():
    iqname = str(input("Enter iqtools name: "))
    #classid = str(input("Enter class id: "))
    name = str(input("Enter name: "))
    password = str(input("Enter pass: "))
 
    #iter_link = f"https://iqtools-{iqname}.intrtl.com/validate?klass_id={classid}&lots_id&size_id&confirm=&page=1&proposal=false&limit=0&klass_search="
    base_link = f"https://iqtools-{iqname}.intrtl.com/validate?confirm=&page=1&proposal=false&limit=0"
 
    driver = webdriver.Chrome(executable_path=r'C:\Users\elili\Documents\Python\chromedriver')
    driver.get (base_link)
    try:
        element_present = EC.presence_of_element_located((By.NAME, 'login-button'))
        WebDriverWait(driver, 10).until(element_present)
    except TimeoutException:
        print("Timed out waiting for page to load")
 
    driver.find_element_by_id("loginform-name").send_keys(name)
    driver.find_element_by_id ("loginform-password").send_keys(password)
    driver.find_element_by_name("login-button").click()
    
    WebDriverWait(driver, 100)
 
    print(driver.page_source)
 
    i = list(range(1,5))
 
    images = driver.find_elements_by_tag_name('img')
        
    for image in images:
        print(image.get_attribute('src'))
    
    while(True):
        pass
 
    #driver.quit()
 
if __name__ == '__main__':
 
    main()

import requests
from lxml import html
import urllib
from bs4 import BeautifulSoup
import io
from seleniumwire import webdriver
import time
from seleniumrequests import Firefox
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.proxy import Proxy, ProxyType
import json
 
#driver = webdriver.PhantomJS()
#options = webdriver.FirefoxOptions()
#options.add_argument('--headless')
#driver = webdriver.Firefox(options=options)
#profile = webdriver.FirefoxProfile()
#profile.accept_untrusted_certs = True
 
#driver = webdriver.Firefox(firefox_profile=profile)
def ajax_complete(driver):
    try:
        return 0 == driver.execute_script("return jQuery.active")
    except WebDriverException:
        return not driver.execute_script("return jQuery && jQuery.active")
 
def main():
    iqname = str(input("Enter iqtools name: "))
    classid = str(input("Enter class id: "))
    name = str(input("Enter name: "))
    password = str(input("Enter pass: "))
 
    iter_url = f"https://iqtools-{iqname}.intrtl.com/validate?klass_id={classid}&lots_id&size_id&confirm=&page=1&proposal=false&limit=0&klass_search="
    base_url = f"https://iqtools-{iqname}.intrtl.com/validate?confirm=&page=1&proposal=false&limit=0"
 
    #profile = webdriver.FirefoxProfile()
    #profile.accept_untrusted_certs = True
    #options = webdriver.FirefoxOptions()
    #options.add_argument('--headless')
 
    
 
    driver = Firefox(executable_path=r'C:\Users\elili\Documents\Python\geckodriver')
    #driver = webdriver.Chrome(executable_path=r'C:\Users\elili\Documents\Python\chromedriver')
    driver.get (base_url)
    try:
        element_present = EC.presence_of_element_located((By.NAME, 'login-button'))
        WebDriverWait(driver, 10).until(element_present)
    except TimeoutException:
        print("Timed out waiting for page to load")
 
    driver.find_element_by_id("loginform-name").send_keys(name)
    driver.find_element_by_id ("loginform-password").send_keys(password)
    driver.find_element_by_name("login-button").click()
    
    try:
        element_present = EC.presence_of_element_located((By.CLASS_NAME, 'klass-wrapper'))
        WebDriverWait(driver, 10).until(element_present)
    except TimeoutException:
        print("Timed out waiting for page to load")
 
    # Create payload
    payload = {
        "confirm": "",
        "klass_id": classid,
        "klass_search": "",
        "limit": "0",
        "lots_id": "null",
        "page": 1,
        "proposal": "false",
        "size_id": "null"
    }
 
    headers = {
        'content-type': 'application/json',
        'referer': iter_url
    }
 
    driver.request('POST', iter_url, data=json.dumps(payload), headers = headers)
    
    WebDriverWait(driver, 20).until(
             ajax_complete,  "Timeout waiting for page to load")
 
    
    classes = driver.find_elements_by_class_name('klass-wrapper')
 
    for clas in classes:
        image = clas.find_element_by_tag_name('img')
        print(image.get_attribute('src'))
        markup = clas.find_element_by_tag_name('a')
        print(markup.get_attribute('href'))
 
    
    #while(True):
    #    pass
 
    #driver.quit()
 
if __name__ == '__main__':
 
    main()

import requests
from lxml import html
import urllib
from bs4 import BeautifulSoup
import io
import os
from seleniumwire import webdriver
import time
from seleniumrequests import Firefox
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.proxy import Proxy, ProxyType
import json
import pandas as pd
 
#driver = webdriver.PhantomJS()
#options = webdriver.FirefoxOptions()
#options.add_argument('--headless')
#driver = webdriver.Firefox(options=options)
#profile = webdriver.FirefoxProfile()
#profile.accept_untrusted_certs = True
 
#driver = webdriver.Firefox(firefox_profile=profile)
def ajax_complete(driver):
    try:
        return 0 == driver.execute_script("return jQuery.active")
    except WebDriverException:
        return not driver.execute_script("return jQuery && jQuery.active")
 
def main():
    iqname = str(input("Enter iqtools name: "))
    classid = str(input("Enter class id: "))
    name = str(input("Enter name: "))
    password = str(input("Enter pass: "))
    page = int(1)
 
    iter_url = f"https://iqtools-{iqname}.intrtl.com/validate?klass_id={classid}&lots_id&size_id&confirm=&page={page}&proposal=false&limit=0&klass_search="
    base_url = f"https://iqtools-{iqname}.intrtl.com/validate?confirm=&page=1&proposal=false&limit=0"
 
    #profile = webdriver.FirefoxProfile()
    #profile.accept_untrusted_certs = True
 
    options = webdriver.FirefoxOptions()
    #options = webdriver.ChromeOptions()
    #options.add_argument('--headless')
 
    driver = Firefox(executable_path=r'C:\Users\elili\Documents\Python\geckodriver', options = options)
    #driver = webdriver.Chrome(executable_path=r'C:\Users\elili\Documents\Python\chromedriver')
    driver.get(base_url)
    try:
        element_present = EC.presence_of_element_located((By.NAME, 'login-button'))
        WebDriverWait(driver, 10).until(element_present)
    except TimeoutException:
        print("Timed out waiting for page to load")
 
    driver.find_element_by_id("loginform-name").send_keys(name)
    driver.find_element_by_id ("loginform-password").send_keys(password)
    driver.find_element_by_name("login-button").click()
    
    try:
        element_present = EC.presence_of_element_located((By.CLASS_NAME, 'klass-wrapper'))
        WebDriverWait(driver, 30).until(element_present)
    except TimeoutException:
        print("Timed out waiting for page to load")
 
 
    driver.get(iter_url)
 
    times = list(range(1,25))
 
    payload = {
        "confirm": "",
        "klass_id": classid,
        "klass_search": "",
        "limit": "0",
        "lots_id": "null",
        "page": 1,
        "proposal": "false",
        "size_id": "null"
    }
 
    headers = {
        'content-type': 'application/json;charset=utf-8',
        'referer': iter_url
    }
 
    driver.request('POST', iter_url, data=json.dumps(payload), headers = headers)
    
    WebDriverWait(driver, 20).until(
        ajax_complete,  "Timeout waiting for page to load")
 
    pagination = driver.find_element_by_class_name('pagination')
    pages =[]
    pages = driver.find_elements_by_class_name('page-item')
    lastpage = pages[-1].find_element_by_tag_name('a').text
    print(lastpage)
 
 
    l = []
    path = os.getcwd()
 
    nums = list(range(40, int(lastpage)))
    for num in nums:
        page = num
        iter_url = f"https://iqtools-{iqname}.intrtl.com/validate?klass_id={classid}&lots_id&size_id&confirm=&page={page}&proposal=false&limit=0&klass_search="
        driver.get(iter_url)
        
        payload = {
            "confirm": "",
            "klass_id": classid,
            "klass_search": "",
            "limit": "0",
            "lots_id": "null",
            "page": 1,
            "proposal": "false",
            "size_id": "null"
        }
 
        headers = {
        'content-type': 'application/json;charset=utf-8',
        'referer': iter_url
        }
 
        driver.request('POST', iter_url, data=json.dumps(payload), headers = headers)
    
        WebDriverWait(driver, 20).until(
            ajax_complete,  "Timeout waiting for page to load")
        
        classes = driver.find_elements_by_class_name('klass-wrapper')
 
        for clas in classes:
            image = clas.find_element_by_tag_name('img')
            imageurl = image.get_attribute('src')
            image = str(imageurl)[str(imageurl).rfind('/'):str(imageurl).rfind("?")]
            urllib.request.urlretrieve(imageurl, path + str(imageurl)[str(imageurl).rfind('/'):str(imageurl).rfind("?")])
            markup = clas.find_element_by_tag_name('a')
            markupurl = markup.get_attribute('href')
            row = [image, markupurl]
            l.append(row)
 
    df = pd.DataFrame(l, columns=["image", "markupurl"])
 
    df.to_excel("validation.xlsx")
    #while(True):
    #    pass
 
    #driver.quit()
 
if __name__ == '__main__':
 
    main()