from selenium import webdriver from selenium.webdriver.common.by import By import time import ssl ssl._create_default_https_context = ssl._create_unverified_context op = webdriver.ChromeOptions() op.add_argument('--no-sandbox') op.add_argument('--disable-notifications') op.add_argument("--lang=en-GB") #op.headless = True driver=webdriver.Chrome( options=op) driver.get('https://www.noon.com/uae-en/beauty/') time.sleep(10) element = driver.find_element(By.CSS_SELECTOR, '.componentArea-9') title = element.find_element(By.CSS_SELECTOR, '.truncate-title-header').text products = element.find_elements(By.CSS_SELECTOR, '.sc-kCMKrZ.ealOXE') urls = [] for product in products: url = product.find_element(By.TAG_NAME, 'a').get_attribute('href') urls.append(url) data = { "title": title, "products": urls } print(data) driver.close()