利用Python爬取高德地图全国地铁站点信息

利���Python中的requests库进行地铁站点信息的获取,同时将数据保存在本机excel中

# 首先引入所需要的包import requestsfrom bs4 import BeautifulSoupimport pandas as pdimport json# 发送 GET 请求获取网页内容url = 'http://map.amap.com/subway/index.html'response = requests.get(url)# 第一步：爬取两个 p 中的城市数据（包括 ID 和拼音），生成城市集合if response.status_code == 200:    # 解码    response_content = response.content.decode('utf-8')    # 使用 Beautiful Soup 解析网页内容    soup = BeautifulSoup(response_content, 'html.parser')    # 从这里开始，你可以使用 Beautiful Soup 对象（soup）来提取所需的信息    # 例如，查找标题    title = soup.title    # 通过Beautiful Soup来找到城市信息元素，并提取这个元素的信息    for soup_a in soup.find('p', class_='city-list fl').find_all('a'):        city_name_py = soup_a['cityname']        city_id = soup_a['id']        city_name_ch = soup_a.get_text()        city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch})        # 获取未显示出来的城市列表    for soup_a in soup.find('p', class_='more-city-list').find_all('a'):        city_name_py = soup_a['cityname']        city_id = soup_a['id']        city_name_ch = soup_a.get_text()        city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch})        print(city_info_list)else:    print("无法获取网页内容")for city_info in city_info_list:    city_id = city_info.get("id")    city_name = city_info.get("name_py")    city_name_ch = city_info.get("name_ch")    print("开始爬取城市" + city_name_ch + "的数据")    city_lines_list = []    # 第二步：遍历城市集合，构造每一个城市的 url,并下载数据    # 构造每个城市的url    url = "http://map.amap.com/service/subway?_1717380520536&srhdata=" + city_id + '_drw_' + city_name + '.json'    res = requests.get(url)    content = res.content.decode('utf-8')    # 将内容字符串转换成json对象    content_json = json.loads(content)    # 提取该城市的所有地铁线list    line_info_list = content_json.get("l")    # 第三步：开始处理每一个地铁线，提取内容到dataframe中    for line_info in line_info_list:        # 地铁线名字        line_name = line_info["kn"]        # 处理地铁线站点        df_per_zd = pd.DataFrame(line_info["st"])        df_per_zd = df_per_zd[['n', 'sl', 'poiid', 'sp', 't', 'su', 'sid']]        df_per_zd['gd经度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[0])        df_per_zd['gd纬度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[1])        df_per_zd.drop('sl', axis=1, inplace=True)        df_per_zd['路线名称'] = line_info['ln']        df_per_zd['城市名称'] = city_name_ch        df_per_zd.rename(columns={"n": "站点名称", "poiid": "POI编号", "sp": "拼音名称", "t": "换乘标志 1:换乘，0:不可换乘", "su": "su", "sid": "sid编号"}, inplace=True)        # 先将这条地铁线处理过的dataframe存起来，我们后面给他放到一张表里        city_lines_list.append(df_per_zd)    # 这段代码就是将地铁线数据列表聚合到一张表里，形成每个城市的地铁站数据    city_subway_data = pd.concat(city_lines_list, ignore_index=True)    # 第四步：将处理好的文件保存为xlsx    city_subway_data.to_excel(city_name_ch + '.xlsx', sheet_name='Sheet1')