2019-07-16 14:16:39 4777瀏覽
本篇文章扣丁學堂Python培訓(xùn)小編實例為大家分享一下Python爬蟲獲取小區(qū)經(jīng)緯度,以及結(jié)構(gòu)化的地址的方法,文中有代碼列出供大家參考,感興趣的小伙伴就來了解一下吧。
通過小區(qū)名稱利用百度api可以獲取小區(qū)的地址以及經(jīng)緯度,但是由于api返回的值中的地址形式不同,所以可以首先利用小區(qū)名稱進行一輪爬蟲,獲取小區(qū)的經(jīng)緯度,然后再利用經(jīng)緯度Reverse到小區(qū)的結(jié)構(gòu)化的地址。
另外小區(qū)名稱如果是'...號‘,可以在爬蟲開始之前在'號‘之后加一個'院‘,得到的精確度更高。這次寫到程序更加便于二次利用,只需要給程序傳遞一個dataframe就可以坐等結(jié)果了?,F(xiàn)在程序已經(jīng)寫好了,就等接下來在工作中看看效果如何了。
class GetAddressInfo:
def __init__(self,df):
import pandas
assert type(df) == pandas.core.frame.DataFrame and ('city' in df.columns) and ('name' in df.columns),\
'The dataframe is not vailid'
from bs4 import BeautifulSoup
from urllib import request
import re
import pandas as pd
import numpy as np
import urllib.parse as urp
self.__data__ = df
def get_address(self):
import numpy as np
self.__data__['小區(qū)經(jīng)度'] = np.nan
self.__data__['小區(qū)緯度'] = np.nan
self.__data__['小區(qū)地址'] = np.nan
for i in self.__data__.index:
self.__data__.loc[i,'小區(qū)緯度'],self.__data__.loc[i,'小區(qū)經(jīng)度'],self.__data__.loc[i,'小區(qū)地址'] =\
self.__get_neigbour_address__(self.__data__.loc[i,'name'],\
self.__data__.loc[i,'city'])
return self.__data__
def __lat__(self,res):
try:
return pd.to_numeric(re.findall('"lat":(.*)',res)[0].split(',')[0])
except:
return 0
def __lng__(self,res):
try:
return pd.to_numeric(re.findall('"lng":(.*)',res)[0])
except:
return 0
def __address__(self,res):
try:
return re.findall('"address":"(.*)",',res)[0]
except:
return 'None'
def __get_neigbour_address__(self,name,city):
my_ak = ##替換自己的ak
qurey = urp.quote(name)
tag = urp.quote('住宅區(qū)')
try:
url = 'http://api.map.baidu.com/place/v2/search?query='+qurey+'&tag='+tag+'®ion='+urp.quote(city)+'&output=json&ak='+my_ak
req = request.urlopen(url)
res = req.read().decode()
lat = self.__lat__(res)
lng = self.__lng__(res)
address = self.__address__(res)
return lat,lng,address
except:
return 0,0,'None'
class ReverseGetAddress:
def __init__(self,data):
assert ('小區(qū)緯度' in data.columns) and ('小區(qū)經(jīng)度' in data.columns) and ('name' in data.columns),\
'The DataFrame is not vailid'
from bs4 import BeautifulSoup
from urllib import request
import re
import pandas as pd
import numpy as np
import urllib.parse as urp
self.__data__ = data
def __get_address1__(self,url):
try:
req = request.urlopen(url)
res = req.read().decode()
address = re.findall('address":"(.*?)"',res)[0]
return address
except:
return 'None1'
def __to_string__(self,arr):
return str(arr)
def __get_address2__(self):
my_ak = ##替換自己的Ak
base_url1 = 'http://api.map.baidu.com/geocoder/v2/?callback=renderReverse'
base_url2 = '&location='
base_url3 = '&pois=0&radius=1&output=json&pois=1&ak='
url = base_url1+base_url2+self.__data__['小區(qū)緯度'].apply(self.__to_string__)+','\
+self.__data__['小區(qū)經(jīng)度'].apply(self.__to_string__)+base_url3+my_ak
return url
def get_address(self):
url = self.__get_address2__()
self.__data__['小區(qū)地址'] = url.apply(self.__get_address1__)
return self.__data__
想要了解更多關(guān)于Python和人工智能方面內(nèi)容的小伙伴,請關(guān)注扣丁學堂Python培訓(xùn)官網(wǎng)、微信等平臺,扣丁學堂IT職業(yè)在線學習教育平臺為您提供權(quán)威的Python開發(fā)環(huán)境搭建視頻,Python培訓(xùn)后的前景無限,行業(yè)薪資和未來的發(fā)展會越來越好的,扣丁學堂老師精心推出的Python視頻教程定能讓你快速掌握Python從入門到精通開發(fā)實戰(zhàn)技能??鄱W堂Python技術(shù)交流群:279521237。
【關(guān)注微信公眾號獲取更多學習資料】 【掃碼進入Python全棧開發(fā)免費公開課】