QHarr

您可以使用css子级 > 和后代 组合符,用于指定具有类的父对象的直接子对象或子对象(分别) search-item ,在一个只分离出感兴趣的HREF的序列中

links = [i.text for i in 
         driver.find_elements_by_css_selector("#search-items > .search-item .details > a")]

因为需要一个用户代理头,不能用 read_html .你可以先拿桌子 requests ,指定适当的标题,然后移交给熊猫:

from pandas import read_html as rh
import requests
from bs4 import BeautifulSoup as bs

r = requests.get('https://finance.yahoo.com/quote/NFLX/history?p=NFLX%27', headers = {'User-Agent':'Mozilla/5.0'})
soup = bs(r.content, 'lxml')
table = rh(str(soup.select_one('[data-test="historical-prices"]')))[0]
print(table)

为什么不找出这些下拉列表的数量,然后将每个下拉列表循环设置为一个变量,并使用通常的语法选择其中的选项?

number_of_dropdowns =  len(WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[id^='msku-sel-']"))))

for i in range(1, number_of_dropdowns + 1):
    current_dropdown = driver.find_element_by_css_selector("[id^='msku-sel-" + str(i) + "']")
    #then use usual syntax to work with select 

Selenium - Python - drop-down menu option value

等待条件的导入:

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

循环示例:

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select

urls = ['https://www.ebay.co.uk/itm/Pashmina-Scarf-100-Viscose-Plain-Wrap-Shawl-Stole-Scarf-Many-Colours-Available/252342060680']
d = webdriver.Chrome()

for url in urls:
    d.get(url)
    number_of_dropdowns =  len(WebDriverWait(d,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[id^='msku-sel-']"))))

    for i in range(1, number_of_dropdowns + 1):
        #then use usual syntax to work with select 
        select = Select(d.find_element_by_css_selector("[id^='msku-sel-" + str(i) + "']"))
        options = select.options
        for index in range(1, len(options) - 1): #avoid first index
            select.select_by_index(index)
            #print(select.first_selected_option.text)
            #your code to extract info per option goes here

您可以使用pandas read_html获取表,然后在klasse列上筛选

import pandas as pd

def get_lectures_two(df, klasse):    
    new_df = df[df['(Klasse(n))'] == klasse]
    return new_df

def get_df(url):
    df = pd.read_html(url)[0]
    df = df[~df['Stunde'].str.contains("LEHRER")]
    return df

df = get_df('https://niwla23.gitlab.io/download/vertreterdemo.html')
print(get_lectures_two(df, '5b'))

在bs4 4.7.1+中,您可以使用:contains和:has,以及通过nth类型的适当列索引来针对适当的行(我在这里使用pandas只是为了快速生成一个很好的表格输出以供查看。。。。您已经拥有来自bs4的列表列表,并且可以使用csv编写示例)

import requests
from bs4 import BeautifulSoup as bs
import pandas as pd

def get_lectures(klasse):
    rows = []
    for row in soup.select(f'.mon_list tr:has(td:nth-of-type(5):contains("{klasse}"))'):
        rows.append([td.text.replace('\xa0','') for td in row.select('td')])
    return rows

r = requests.get('https://niwla23.gitlab.io/download/vertreterdemo.html')
soup = bs(r.content, 'lxml')
headers = [th.text for th in soup.select('th.list')]
klasse = '5b'

df = pd.DataFrame(get_lectures(klasse), columns = headers)
print(df)

您需要driver.find_元素(如果有多个元素)。这将返回一个列表。对于css选择器,您要确保为具有子href的类选择

elems = driver.find_elements_by_css_selector(".sc-eYdvao.kvdWiq [href]")
links = [elem.get_attribute('href') for elem in elems]

您可能还需要等待css选择器找到的所有元素的出现。

elems = WebDriverWait(driver,10.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".sc-eYdvao.kvdWiq [href]")))

您可以尝试使用控制汇总,方法是提取结果汇总并将特色汇总添加到其中。这些数字已经提供给您,因此您可以循环直到hrefs满足这个要求。您可能想在循环中添加超时。

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')

prefs = {'profile.managed_default_content_settings.images':2}
chrome_options.add_experimental_option("prefs", prefs)
driver = webdriver.Chrome(chrome_options=chrome_options) 
driver.get("http://biggestbook.com/ui/catalog.html#/search?cr=1&rs=12&st=BM&category=1")
wait = WebDriverWait(driver,20)
nonFeaturedTotal = int(wait.until(EC.presence_of_element_located((By.CSS_SELECTOR , '.ess-view-item-count-text'))).text.split(' ')[-1])
featuredTotal = len(wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".ess-product-container-featured"))))
expectedTotal = featuredTotal + nonFeaturedTotal

while False:
    len(driver.find_elements_by_css_selector(".ess-product-brand + [href]")) == expectedTotal

links = driver.find_elements_by_css_selector(".ess-product-brand + [href]")
results = [link.get_attribute("href") for link in links]

print(len(results))
print(links)

driver.quit()

分10批进行,更改url中的起始值。可以循环递增和添加add变量

https://www.indeed.co.in/jobs?q=software+developer&l=Bengaluru%2C+Karnataka&start=0

https://www.indeed.co.in/jobs?q=software+developer&l=Bengaluru,+Karnataka&start=1

例如。

import requests
from bs4 import BeautifulSoup as bs
import pandas as pd
results = []
url = 'https://www.indeed.co.in/jobs?q=software+developer&l=Bengaluru,+Karnataka&start={}'
with requests.Session() as s:
    for page in range(5):
        res = s.get(url.format(page))
        soup = bs(res.content, 'lxml')
        titles = [item.text.strip() for item in soup.select('[data-tn-element=jobTitle]')]
        companies = [item.text.strip() for item in soup.select('.company')]
        data = list(zip(titles, companies))
        results.append(data)
newList = [item for sublist in results for item in sublist]
df = pd.DataFrame(newList)
df.to_json(r'C:\Users\User\Desktop\data.json')

以下是我将如何为所有地区。需要bs4 4.7.1。AFAICS你必须假设公司的顺序是一致的。

import requests
from bs4 import BeautifulSoup
import pandas as pd

r = requests.get("https://opensignal.com/reports/2019/04/uk/mobile-network-experience")
soup = BeautifulSoup(r.content,'lxml') #'html.parser' if lxml not installed
metrics = ['4g-availability', 'video-experience', 'download-speed' , 'upload-speed', 'latency']
headers = ['02', 'Vodaphone', '3', 'EE']
results = []

for region in soup.select('.s-regional-analysis__region'):
    for metric in metrics:
        providers = [item.text for item in region.select('.c-ru-chart:has([data-metric="' + metric + '"]) .c-ru-graph__number')]
        row = {headers[i] : providers[i] for i in range(len(providers))}
        row['data-metric'] = metric
        row['region'] = region['id'] 
        results.append(row)

df = pd.DataFrame(results, columns = ['region', 'data-metric', '02','Vodaphone', '3', 'EE'] )
print(df)

样本输出:

页面使用lat和lon来请求数据。您可以模拟xhr(首先获取传递位置的lat和lon,我使用 free API 为了这个。你怎么做取决于你自己。)

您可以在这里看到请求:

响应json包含针对键的html行。输出示例

由于与行中键相关联的内部值是html,所以我将其传递回BeautifulSoup进行处理。html内容示例:

import requests
import json
import pandas as pd
from bs4 import BeautifulSoup as bs
apiKey = "yourFreeAPIkey"
address = "78759" 

url = "https://api.opencagedata.com/geocode/v1/json?q=" + address + "&key=" + apiKey + "&pretty=1"
res = requests.get(url).json()
data = res['results'][1]['geometry']
lat = data['lat']
lng = data['lng']
date = '2019-03-09'
res = requests.get('https://www.imax.com/showtimes/ajax/theatres?date=' + date + '&lat=' + str(lat) + '&lon=' + str(lng))
soup = bs(res.content, 'lxml')
newData = json.loads(soup.select_one('p').text)
columns = ['movieTitle', 'movieLink', 'theatreLink', 'address','movieFormat', 'times']
baseURL = 'https://www.imax.com'
results = []

for row in newData['rows']:
    soup = bs(row['row'], 'lxml')
    link = baseURL + soup.select_one('a')['href']
    address = soup.select_one('.theatre-address').text.strip()
    movieTitle = soup.select_one('.movie-title').text.strip()
    movieLink = baseURL + soup.select_one('.movie-title a')['href']
    movieFormat = soup.select_one('.movie-format').text.strip()
    times = [item.text.strip() for item in soup.select('.line-items a')]
    results.append([movieTitle, movieLink, link, address, movieFormat, times])
df = pd.DataFrame(results, columns = columns)
print(df)

示例结果:

要添加到已经给出的答案中,您可以从初始json获得总结果。然后可以使用循环来增加批处理

import requests
import json

url = "https://clinicaltrialsapi.cancer.gov/v1/clinical-trials"
r = requests.get(url).json()
num_results = int(r['total'])
results_per_request = 50
total = 0
while total < num_results:
    total+=results_per_request
    print(total)

以下是一些取决于整个html的可能性

#class selector

driver.find_element_by_css_selector('.pricingSummary-priceList-value').text.replace('$','')

#attribute selector
driver.find_element_by_css_selector('[itemprop=youSave]').text.replace('$','')

#combined
driver.find_element_by_css_selector('[itemprop=youSave].pricingSummary-priceList-value').text.replace('$','')

#combined with type selector
driver.find_element_by_css_selector('dd[itemprop=youSave].pricingSummary-priceList-value').text.replace('$','')

如果需要多个匹配项,则使用列表理解示例( find_elements 返回列表):

[i.text.replace('$','') for i in driver.find_elements_by_css_selector('[itemprop=youSave]')]

类比类型快,类型比属性快。在某种程度上,短组合比长组合更快,这取决于之前给出的规则和顺序,但这超出了本文的范围。

我使用了一个多值类的单个类。你可以在其他类中添加

driver.find_element_by_css_selector('.saleprice.pricingSummary-priceList-value').text.replace('$','')

更重要的是,您可以使用或css选择器语法来组合pdf、docx等。注意,您仍然需要完成一些路径,例如带有前缀 "http://www.soc.napier.ac.uk/~40009856/CW/" . 以下用途 attribute = value 带$operator的css选择器语法(这意味着属性字符串的值以结尾)

from bs4 import BeautifulSoup
import requests
url= 'http://www.soc.napier.ac.uk/~40009856/CW/'
res = requests.get(url)
soup = BeautifulSoup(res.content, 'lxml')
items = soup.select("[href$='.docx'], [href$='.pdf'], img[src]")
print([item['href'] if 'href' in item.attrs.keys()  else item['src'] for item in items])

实际上,您正在跟踪以下链接(单击时),这样您就可以收集链接,然后使用.get循环。使用 return [...document.querySelectorAll ] 句法

from selenium import webdriver

d = webdriver.Chrome()
d.get("https://www.quora.com/profile/Ashley-Isabella-3")
links = d.execute_script("return [...document.querySelectorAll('.ui_qtext_more_link')].map(link => link['href'])")
info = []
for link in links:
    d.get(link)
    pageInfo = d.execute_script("return [...document.querySelectorAll('.ui_qtext_para')].map(link => link.innerText)")
    info.append(pageInfo)
print(info)

我认为你可以做到以下几点:

那是css选择器

find_element_by_css_selector(".gradient .delete-tab").click()

这将匹配的元素限制为一个

在你的代码中:

except:
    element_delete = driver.find_element_by_css_selector('.gradient .delete-tab, .btn-delete')
    element_delete.click()

这意味着你可以把这个移到 try 相反,它已经尝试匹配两个选择器。

如果它是所示的最后一个子对象,则可以使用最后一个子选择器

.col-md-6:last-child .ibox-content div

你也可以用第N种

.col-md-6:nth-of-type(n) .ibox-content div

甚至是最后一种

.col-md-6:last-of-type .ibox-content div

你可以把硒和BS4结合起来。

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import math
from bs4 import BeautifulSoup as bs
import pandas as pd

def getData(properties):
    for aProperty in properties:
        soup = bs(str(aProperty) , 'lxml')
        price = soup.select_one('.property-price-heading').text if soup.select_one('.property-price-heading') is not None else ''       
        header = soup.select_one('.property-marketing-heading').text if soup.select_one('.property-marketing-heading') is not None else ''        
        beds = soup.select_one('.icon-beds span').text if soup.select_one('.icon-beds span') is not None else ''        
        baths = soup.select_one('.icon-baths span').text if soup.select_one('.icon-baths span') is not None else ''    
        garages = soup.select_one('.icon-garages span').text if soup.select_one('.icon-garages span') is not None else ''
        land = soup.select_one('.property-list-land-size .value').text if soup.select_one('.property-list-land-size .value') is not None else ''
        description = soup.select_one('.results-content').text.strip() if soup.select_one('.results-content') is not None else ''
        ref = soup.select_one('.property-list-webref').text.strip() if soup.select_one('.property-list-webref') is not None else ''
        items = [price, header, beds, baths, garages, land, description, ref ]
        result.append(items)
    return result

varUrl = 'https://www.jawitz.co.za/results/residential/for-sale/cape-town/all/?p={}&advanced_search=1&s=-price'
url = 'https://www.jawitz.co.za/results/residential/for-sale/cape-town/all/?advanced_search=1'

driver = webdriver.Chrome()
driver.get(url)
data =  WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".property-list-details")))
numResults = int(driver.find_element_by_id('id_property_count').text.split(' ')[0])

soup = bs(driver.page_source, 'lxml')
properties = soup.select('.property-list-details')

resultsPerPage = 10
numPages = math.ceil(numResults/resultsPerPage)

results = []

results.append(getData(properties))

if numPages > 1:
    for page in range(2, numPages + 1):
        url = varUrl.format(page)
        driver.get(url)
        data =  WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".property-list-details")))
        soup = bs(driver.page_source, 'lxml')
        properties = soup.select('.property-list-details')
        results.append(getData(properties))
        if page > 3:  #delete after testing
            break  #delete after testing

finalList = [item for sublist in results for item in sublist]
print(finalList)