爬虫实战-爬取天气预报

import requests
from lxml import etree
import time
import re

city = 'hangzhou'
index_url = 'http://www.tianqihoubao.com/lishi/{}.html'

index_page = requests.get(index_url.format(city))
index_html = etree.HTML(index_page.text)

year_ele = index_html.xpath('//*[@id="content"]/div[@class="box pcity"]')
temp_m = []
for year in year_ele:
    for month in year.xpath('ul/li/a'):
        month_id = month.xpath('text()')[0]
        month_url = month.xpath('@href')[0]
        temp_m.append([month_id,month_url])


def month_range(start='',end=''):
    month_list = []
    while int(start[:4])<int(end[:4]):
        month_list.append(start)
        if start[4:6] == '12':
            start = str(int(start[:4]) + 1) + '01'
        else:
            start = str(int(start) + 1)
    while int(start) < int(end):
        month_list.append(start)
        start = str(int(start) + 1)
    month_list.append(end)
    return month_list


temp_d = []
url = 'http://www.tianqihoubao.com/lishi/{0}/month/{1}.html'

for year in month_range('201904','201904'):
    print(year,'天气数据下载中……')
    temp_page = requests.get(url.format(city, year))
    print(year,'数据下载成功')
    temp_html = etree.HTML(temp_page.text)
    temp_ele = temp_html.xpath('//*[@class="wdetail"]/table/tr')
    print(temp_ele)
    for day in temp_ele[1:]:
        date_time = day.xpath('td[1]/a/text()')[0].strip()
        print(date_time, end=' ')
        date_id = ''.join(re.findall('(.*?)年(.*?)月(.*?)日',date_time)[0])
        print(date_id, end=' ')
        date_weth = day.xpath('td[2]/text()')[0].strip().replace(' ','').replace('\r\n', '')
        print(date_weth, end=' ')
        date_weth_day = date_weth.split('/')[0]
        print(date_weth_day,end=' ')
        date_weth_night = date_weth.split('/')[1]
        print(date_weth_night, end=' ')
        date_temp = day.xpath('td[3]/text()')[0].strip().replace(' ','').replace('\r\n', '')
        print(date_temp, end=' ')
        date_temp_day = date_temp.replace('℃', '').split('/')[0]
        print(date_temp_day, end=' ')
        date_temp_night = date_temp.replace('℃', '').split('/')[1]
        print(date_temp_night, end=' ')
        date_windlv = day.xpath('td[4]/text()')[0].strip().replace(' ','').replace('\r\n', '')
        print(date_windlv, end=' ')
        date_windlv_day = date_windlv.split('/')[0]
        print(date_windlv_day, end=' ')
        date_windlv_night = date_windlv.split('/')[1]
        print(date_windlv_night, end=' ')
        temp_d.append([date_id, date_weth_day, date_weth_night, date_temp_day,
                       date_temp_night, date_windlv_day, date_windlv_night, date_id[:4], date_id[4:6]])
        print()
    time.sleep(10)

from lxml import etree
import ssl
import requests

index_page = requests.get('http://www.tianqihoubao.com/yubao/hangzhou.html')
#print(index_page)
index_html = etree.HTML(index_page.text)
hztq = index_html.xpath('//*[@class="wdetail"]/table/tr')
print(hztq)
num = 0
for day in hztq[1:]:
    if num % 2 == 0:
        day_time = day.xpath('td[1]/text()')[0].strip()
        print(day_time)
        day_status = day.xpath('td[2]/text()')[0].strip()
        print(day_status, end='\t')
        day_weather = day.xpath('td[3]/b/text()')[0].strip()
        print(day_weather, end='\t')
        day_high = day.xpath('td[4]/b/text()')[0].strip()
        print(day_high, end='\t')
        day_rain = day.xpath('td[5]/text()')[0].strip()
        print(day_rain, end='\t')
    else:
        day_status = day.xpath('td[1]/text()')[0].strip()
        print(day_status, end='\t')
        day_weather = day.xpath('td[2]/b/text()')[0].strip()
        print(day_weather, end='\t')
        day_high = day.xpath('td[3]/b/text()')[0].strip()
        print(day_high, end='\t')
        day_rain = day.xpath('td[4]/text()')[0].strip()
        print(day_rain, end='\t')
    num += 1
    print('')