import requests
import random
url = "https://hk.trip.com/flights/Ajax/SearchFlight"
payload = "------WebKitFormBoundary7MA4YWxkTrZu0gW\r\nContent-Disposition: form-data; name=\"context\"\r\n\r\n{\"SearchNo\":\"1\",\"FlightWay\":\"OW\",\"SubChannel\":\"0\",\"SearchToken\":\"1\",\"Class\":\"Y\",\"SegmentParameterList\":[{\"DCityCode\":\"tyo\",\"ACityCode\":\"bjs\",\"DDateString\":\"2018-07-01\"}],\"TravelerParameteList\":[{\"TravelerType\":\"ADT\",\"TravelerCount\":\"3\"},{\"TravelerType\":\"CHD\",\"TravelerCount\":\"2\"},{\"TravelerType\":\"INF\",\"TravelerCount\":\"0\"}]}\r\n------WebKitFormBoundary7MA4YWxkTrZu0gW--"
USER_AGENTS = [
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:57.0) Gecko/20100101 Firefox/57.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'
]
headers = {
"authority":"hk.trip.com","method":"POST","path":"/flights/Ajax/SearchFlight","scheme":"https","accept":"application/json, text/javascript, */*; q=0.01","accept-encoding":"gzip, deflate, br","accept-language":"zh-CN,zh;q=0.9","content-length":"538","content-type":"application/x-www-form-urlencoded; charset=UTF-8","cookie":"_RSG=7tavgNEymoFF9uJ.C3HDz8; _RDG=2893f045301e2327e21f31a48427a84c68; _RGUID=fcb0f0e5-eae6-4dc7-8f4d-44333e4f824f; ibulocale=zh_hk; __utmz=1.1529635752.1.1.utmcsr=trip.com|utmccn=(referral)|utmcmd=referral|utmcct=/; ibu_h5_group=trip; ibu_h5_site=HK; ibu_h5_lang=hk; ibu_h5_local=zh-hk; _tp_search_latest_channel_name=flights; _abtest_userid=ed1c247e-91ef-4053-825b-027a49ddfb22; DUID=u=A50A20DDFEC07ACCD66993A0E6AE7159&v=0; MKtUnionTM=1530241151352; _ga=GA1.2.1376959090.1529635752; _gid=GA1.2.1828801722.1529895645; Union=AllianceID=4897&SID=155952&OUID=baidu81|index|||; ibulanguage=HK; ASP.NET_SessionId=fm5uwygkxs3gagjpecmzytzx; __utmc=1; LastSearchDDate=2018-07-01; _RF1=47.52.159.144; cookiePricesDisplayed=CNY; ibu_h5_curr=CNY; _combined=transactionId=20180626165724186; __utma=1.1376959090.1529635752.1529997438.1530004214.17; __utmt=1; _bfi=p1=10320667452&p2=10320667452&v1=151&v2=150; ASP.NET_SessionSvc=MTAuMTUuMTI4LjIzfDkwOTB8b3V5YW5nfHwxNTI2NTQwNzQyNjI5; _combined=transactionId=20180626172933104; __utmb=1.4.10.1530004214; _bfa=1.1529635684985.43dvn5.1.1530003415174.1530004214449.20.152.10320667439; _bfs=1.4; _uetsid=_ueta431f353","origin":"https://hk.trip.com","referer":"https://hk.trip.com/flights/tokyo-to-beijing/tickets-tyo-bjs/?flighttype=s&dcity=tyo&acity=bjs&startdate=2018-07-01&searchboxarg=t&class=y&quantity=3&childqty=2&searchboxarg=t","user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36","x-requested-with":"XMLHttpRequest",
'content-type': "multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW",
'cache-control': "no-cache",
'postman-token': "b5cce59e-9e46-9ac4-5bcd-2aeb40bb46df"
}
headers['user-agent'] = random.choice(USER_AGENTS)
response = requests.request("POST", url, data=payload, headers=headers ,proxies =proxies,verify=False)
response = response.json()
# print(response)
print(len(response['fltProductInfoList']))
爬取trip網(wǎng)站機(jī)票信息
例如 正常返回是91條數(shù)據(jù)
但是 幾分鐘之后就被網(wǎng)站做了限制 只返回33條
嘗試 random user-agent proxies 均無效果
想問下 如何破解
北大青鳥APTECH成立于1999年。依托北京大學(xué)優(yōu)質(zhì)雄厚的教育資源和背景,秉承“教育改變生活”的發(fā)展理念,致力于培養(yǎng)中國IT技能型緊缺人才,是大數(shù)據(jù)專業(yè)的國家
北大青鳥中博軟件學(xué)院創(chuàng)立于2003年,作為華東區(qū)著名互聯(lián)網(wǎng)學(xué)院和江蘇省首批服務(wù)外包人才培訓(xùn)基地,中博成功培育了近30000名軟件工程師走向高薪崗位,合作企業(yè)超4
中公教育集團(tuán)創(chuàng)建于1999年,經(jīng)過二十年潛心發(fā)展,已由一家北大畢業(yè)生自主創(chuàng)業(yè)的信息技術(shù)與教育服務(wù)機(jī)構(gòu),發(fā)展為教育服務(wù)業(yè)的綜合性企業(yè)集團(tuán),成為集合面授教學(xué)培訓(xùn)、網(wǎng)
達(dá)內(nèi)教育集團(tuán)成立于2002年,是一家由留學(xué)海歸創(chuàng)辦的高端職業(yè)教育培訓(xùn)機(jī)構(gòu),是中國一站式人才培養(yǎng)平臺(tái)、一站式人才輸送平臺(tái)。2014年4月3日在美國成功上市,融資1
曾工作于聯(lián)想擔(dān)任系統(tǒng)開發(fā)工程師,曾在博彥科技股份有限公司擔(dān)任項(xiàng)目經(jīng)理從事移動(dòng)互聯(lián)網(wǎng)管理及研發(fā)工作,曾創(chuàng)辦藍(lán)懿科技有限責(zé)任公司從事總經(jīng)理職務(wù)負(fù)責(zé)iOS教學(xué)及管理工作。
浪潮集團(tuán)項(xiàng)目經(jīng)理。精通Java與.NET 技術(shù), 熟練的跨平臺(tái)面向?qū)ο箝_發(fā)經(jīng)驗(yàn),技術(shù)功底深厚。 授課風(fēng)格 授課風(fēng)格清新自然、條理清晰、主次分明、重點(diǎn)難點(diǎn)突出、引人入勝。
精通HTML5和CSS3;Javascript及主流js庫,具有快速界面開發(fā)的能力,對(duì)瀏覽器兼容性、前端性能優(yōu)化等有深入理解。精通網(wǎng)頁制作和網(wǎng)頁游戲開發(fā)。
具有10 年的Java 企業(yè)應(yīng)用開發(fā)經(jīng)驗(yàn)。曾經(jīng)歷任德國Software AG 技術(shù)顧問,美國Dachieve 系統(tǒng)架構(gòu)師,美國AngelEngineers Inc. 系統(tǒng)架構(gòu)師。