Scrapy仅对URL执行GET请求，它不是Web浏览器，因此无法运行JavaScript。因此，仅Scrapy不足以抓取动态网页。
另外，您将需要Selenium之类的东西，它基本上为您提供了多个Web浏览器及其功能的接口，其中之一是能够运行JavaScript并获取客户端生成的HTML。
以下是如何执行此操作的摘要：
from Project.items import SomeItem
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.selector import Selector
from selenium import webdriver
import time
class RandomSpider(CrawlSpider):
name = 'RandomSpider'
allowed_domains = ['random.com']
start_urls = [
'http://www.random.com'
]
rules = (
Rule(SgmlLinkExtractor(allow=('some_regex_here')), callback='parse_item', follow=True),
)
def __init__(self):
CrawlSpider.__init__(self)
# use any browser you wish
self.browser = webdriver.Firefox()
def __del__(self):
self.browser.close()
def parse_item(self, response):
item = SomeItem()
self.browser.get(response.url)
# let JavaScript Load
time.sleep(3)
# scrape dynamically generated HTML
hxs = Selector(text=self.browser.page_source)
item['some_field'] = hxs.select('some_xpath')
return item
602392714
清零编程群