import scrapy
class LOLHeroNameSpiderItem(scrapy.Item):
'''继承自scrapy.Item，在spiders/lol-hero-name-spidder.py中使用'''
# name英雄名称，e_name英雄英文名称（即英雄id），title英雄称号
    hero_name = scrapy.Field()
    hero_e_name = scrapy.Field()
    hero_title = scrapy.Field()

import codecs
import json
class LOLHeroNamePipeline(object):
    def __init__(self):
        # codecs,python编解码器，以字节写方式打开一个文件，方便后面的转换中文
        # 关于codecs可以查看https://docs.python.org/3/library/codecs.html
        self.file = codecs.open('lol-hero-name.json', 'wb', encoding='utf-8')
    def process_item(self, item, spider):
        line = json.dumps(dict(item)) + '\n'  # line是str类型
        # 向codecs("wb")打开的文件写入line经过utf-8编码后再经过unicode编码，最后存储的字符是中文
        self.file.write(line.encode('utf-8').decode('unicode_escape'))
        return item

from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.selector import Selector
from scrapy_splash import SplashRequest  # 引入splash组件，获取js数据

from lolSpider.items import LOLHeroNameSpiderItem


class LOLHeroNameSpider(Spider):
    '''
    第一个爬虫类，爬取英雄联盟所有英雄名字及称号
    '''    
    name = "LOL-Hero-Name"  # 唯一的名字，命令行执行的时候使用
    # 同一个项目下，多个spider对应不同的pipeline，需要在spider中修改配置，setting.py中不做pipeline配置
    custom_settings = {
        'ITEM_PIPELINES':{
            'lolSpider.pipelines.LOLHeroNamePipeline': 300,
        },        
    }

    allowed_domains = ["http://lol.qq.com/web201310/info-heros.shtml"]  # 允许爬取的网站域
    start_urls = ["http://lol.qq.com/web201310/info-heros.shtml",]  # 开始爬取的第一条url

    def start_requests(self):
        for url in self.start_urls:
            # 处理start_urls，以splashRequest的方法执行parse方法
            yield SplashRequest(url=url, callback=self.parse, args={'wait': 0.5},endpoint='render.html',)

    # parse方法用于处理数据，传入一个response对象
    def parse(self, response):
        # sites是一个可迭代对象，它代表了从response对象中获取某些数据而形成的可迭代对象
        # 使用xpath方法找到特定的元素组成的数据，在这里代表“找到一个id为jSearchHeroDiv的元素它的所有li元素，很多个li元素，所以成为一个可迭代对象
        # XPath是一门在XML、HTML文档中查找信息的语言，它可以查找元素。教程可见http://www.w3school.com.cn/xpath/index.asp
        # 之前的xpath很长，但不需要对着网页源代码查找，请见http://www.locoy.com/Public/guide/V9/HTML/XPath%E6%8F%90%E5%8F%96.html
        sites = response.xpath("//*[@id=\"jSearchHeroDiv\"]/li")
        items = []

        # 对sites的每一个li元素进行循环
        for site in sites:
            item = LOLHeroNameSpiderItem()
            item['hero_name'] = site.xpath("a/@title").extract_first().split(" ")[1]
            item['hero_title'] = site.xpath("a/@title").extract_first().split(" ")[0]  
            item['hero_e_name'] = site.xpath("a/@href").extract_first()[21:]

            yield {'name':item['hero_name'],'title':item['hero_title'],"e_name":item['hero_e_name']}  
            items.append(item)
        # return 也可以完成yield的任务（在这个文件内），两者都可以输出到json文件（保留一个就好hh）
        return items

class LOLHeroSkinSpiderItem(scrapy.Item):
    # image_urls图片链接，image_names图片名称，image_id图片保存名称
    image_urls = scrapy.Field()
    image_names = scrapy.Field()
    image_id = scrapy.Field()

import json
import codecs
from scrapy import log
from scrapy import Request
from scrapy.http import Request
from scrapy.exceptions import DropItem
from scrapy.pipelines.images import ImagesPipeline

class LOLHeroSkinInfoPipeline(object):
    '''
    pipelines是对spider爬取到的item进行处理的过程
    关于scrapy的核心架构可参见https://blog.csdn.net/u012150179/article/details/34441655
    '''
    def __init__(self):
        # codecs,python编解码器，以字节写方式打开一个文件，方便后面的转换中文
        # 关于codecs可以查看https://docs.python.org/3/library/codecs.html
        self.file = codecs.open('lol-skin-info.json', 'wb', encoding='utf-8')
    def process_item(self, item, spider):
        line = json.dumps(dict(item)) + '\n'  # line是str类型
        # 向codecs("wb")打开的文件写入line经过utf-8编码后再经过unicode编码，最后存储的字符是中文
        self.file.write(line.encode('utf-8').decode('unicode_escape'))
        return item


# 对应LOL-Hero-Skin
class LOLHeroSkinPipeline(ImagesPipeline):
    # file_path()方法将保存的图片文件重命名，返回str
    def file_path(self, request, response=None, info=None):
        image_guid = request.url.split('/')[-1]
        return 'full/%s' % (image_guid)

    # 参加https://www.jianshu.com/p/c12d2ac7d55f
    def get_media_requests(self, item, info):
        for image_url in item['image_urls']:
            yield Request(image_url)

    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("Item contains no images")
        return item

# -*- coding:utf-8 -*-
import sys
import scrapy
from scrapy import Request
from scrapy.spiders import Spider
from scrapy_splash import SplashRequest

from lolSpider.items import LOLHeroSkinSpiderItem


class LOLHeroSkinSpider(scrapy.Spider):
    '''第五个爬虫类，爬取所有英雄皮肤'''
    name = 'LOL-Hero-Skin'
    custom_settings = {
        'ITEM_PIPELINES':{
            'lolSpider.pipelines.LOLHeroSkinPipeline': 300,
            'lolSpider.pipelines.LOLHeroSkinInfoPipeline':301,
        },
        'IMAGES_STORE':'../lolSpider/lolSpider/img/hero_skin_img',
    }
    # 从英雄列表中获取英文名，构造start_urls
    start_urls = []    
    base_url = "http://lol.qq.com/web201310/info-defail.shtml?id="
    with open('./lol-hero-name.json','r') as f:
        for line in f.readlines():            
            i = line.find("e_name") +10
            suf_url = line[i:-3]
            start_urls.append(base_url+suf_url)

    # http header    
    default_headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, sdch, br',
        'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Host': 'http://lol.qq.com/web201310/info-defail.shtml?id=Aatrox',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
    }

    def start_requests(self):
        for url in self.start_urls:
            yield SplashRequest(url=url, callback=self.parse, args={'wait': 0.5},
                endpoint='render.html'
            )

    def parse(self,response):
        text = response.xpath("//*[@id=\"skinNAV\"]/li")
        for i in text:
            item = LOLHeroSkinSpiderItem()
            # 获得图片id，为了进一步的重命名
            item["image_id"] = i.xpath("a/img/@src").extract_first().split("/")[-1].replace("small","big")
            # 动态页面，无法抓取到bigxxx.jpg，但可以通过replace构造完整url
            image_urls = i.xpath("a/img/@src").extract_first()
            image_urls = image_urls.replace("small","big")
            temp = []
            temp.append(image_urls)
            item["image_urls"] = temp
            # 皮肤的中文名称，若为“默认”，则添加英雄名称            
            image_names = i.xpath("a/@title").extract_first() 
            if image_names == "默认皮肤":
                item['image_names'] = image_names + " " + response.xpath("//*[@id=\"DATAnametitle\"]/text()").extract_first().split(' ')[-1]
            else:
                item['image_names'] = image_names

            yield item

SPLASH_URL = 'http://172.17.0.2:8050/'  # splash在docker下的url，由机器而异，查找ip可参见https://mozillazg.com/2016/01/docker-get-containers-ip-address.html
# 下载中间件
DOWNLOADER_MIDDLEWARES = {
    'scrapy_splash.SplashCookiesMiddleware': 723,
    'scrapy_splash.SplashMiddleware': 725,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}
# 爬虫中间件
SPIDER_MIDDLEWARES = {
    'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
}
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'  # 去重过滤器（必须）
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage' # 使用http缓存 

ROBOTSTXT_OBEY = True

DOWNLOAD_DELAY = 1

DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}

COOKIES_ENABLED = False