使用xpath和lxml爬取伯乐在线

一、爬取的代码

• 1、网站地址

• 2、具体实现代码

  import requests
  from lxml import etree
  
  class JobBole(object):
      def __init__(self):
          self.url = 'http://python.jobbole.com/all-posts/'
          self.headers = {
              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36',
          }
  
      @property
      def get_article_link(self):
          """
          抓取网页
          :return:
          """
          response = requests.get(url=self.url, headers=self.headers)
          if response.status_code == 200:
              html = etree.HTML(response.text)
              # 现在就取第一个的访问详情页面
              article = html.xpath('//div[@id="archive"]/div[@class="post floated-thumb"]')[0]
              # 查找到当前标签上的a标签
              a_link = article.xpath('./div[@class="post-meta"]/p[@class="align-right"]//a/@href')[0]
              return a_link
  
      def get_html(self):
          """
          获取详情页面的数据
          :return:
          """
          if self.get_article_link:
              response = requests.get(url=self.get_article_link, headers=self.headers)
              if response.status_code == 200:
                  html = etree.HTML(response.text)
                  title = html.xpath('//div[@class="entry-header"]/h1/text()')[0]
                  article = html.xpath('//div[@id="article_content"]')[0]
                  print({"title": title, "article": etree.tostring(article, encoding="utf8").decode("utf8")})
  
  if __name__ == '__main__':
      jobbole = JobBole()
      jobbole.get_html()