本文所有代码都已经发布在github项目主页上

https://github.com/smityliu/spider

微博模拟登录、推特模拟登录

久等了各位伙伴们，本篇文章是我们社交网络人物画像的第三篇，也是我们爬虫最技术含量最为丰富的一篇，之前的两篇文章基于社交网络爬虫分析人物兴趣属性（一）;基于社交网络爬虫分析人物兴趣属性(二)发表了以后，有师傅问过我为什么不做一下模拟登录，会比较好，然后也给了我很多建议，比如如何做数据对齐的建议，看来其实人物画像真的很多师傅有经验，现在大公司也做大数据分析的工作。

我没有直接公开模拟登录的研究过程是因为之前爬虫违规使用的问题闹得沸沸扬扬，所以我不太想在公开平台讨论国内的社交网络，但是后来看见大部分的论坛其实都有公开了新浪微博或者知乎一类的模拟登录，并且我也询问了很多白帽子的同僚，知道了其实正常使用模拟登录是允许的。下面就来进入今天大家都很想看的主题——新浪微博模拟登录和推特模拟登录。

为什么需要这个呢，因为我们遇到了两个问题：

1. 我们之前只选取了一个社交网络，数据偶然性太大了，得选取两个——微博和推特

2. 两个网路哦就必须要做数据对齐，如何保证你做数据融合的两个用户是同一个人，必须得找到相关得信息

在数据对齐的时候要搜索足够多的用户相关信息，其中某用户关注的人员就是很重要的一块，但是我们如果不登陆是看不到他们的关注的：

递归的模型是

很惊喜的地方在于

  headers={  'Host': 'passport.weibo.cn',  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0',  'Accept': '*/*',  'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',  'Content-Type': 'application/x-www-form-urlencoded',  'Origin': 'https://passport.weibo.cn',  'Connection': 'close',  'Referer': 'https://passport.weibo.cn/signin/login?entry=mweibo&r=https%3A%2F%2Fweibo.cn%2F&backTitle=%CE%A2%B2%A9&vt=',  'Cookie':'你的cookie'  }  data={         'username': '你的用户名',              'password': '你的密码',              'savestate': '1',              'r': '',              'ec': '0',              'pagerefer': 'http://weibo.cn/',              'entry': 'mweibo',              'wentry': '',              'loginfrom': '',              'client_id': '',              'code': '',              'qq': '',              'mainpageflag': '1',              'hff': '',              'hfp': '',  }

login=session.post(url=url,headers=headers,data=data)  if("20000000" not in login.text):    print("登陆失败")    exit()  login.encoding='utf-8'

def follow_spider(id_all,turn):  #控制迭代次数  if(turn == 2):    print('\n'+"已经达到需要爬取的层数，准备退出......")    exit()  else:    #最外层的for，是用来循环这一层函数中id_all的值    print('第'+str(turn)+'层开始：'+'\n'+'该层需要爬取的id有：')    print(id_all)    uid_all=[]
    for id in id_all:      people=0      time.sleep(5)      #登录以后使用新的headers      headers_logined={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0'}
      #先标记名字id      url_for_name='https://weibo.cn/'+id      try:        l=session.get(url=url_for_name,headers=headers_logined,timeout=3)        name=re.findall(r'<title>(.*?)的微博</title>',l.text)        print('\n'+'开始爬取'+id+'||'+name[0]+'的所有关注\n')      except:        continue      follow_url= 'https://weibo.cn/'+str(id)+'/follow'      login=session.get(url=follow_url,headers=headers_logined,timeout=3)      pages = re.findall(r'value="跳页" />&nbsp;1\/(.*?)页',login.text)      #print(pages)      if len(pages)==0:        print('\n'+"只有一页，以下是"+id+'||'+name[0]+'的所有关注')        page_all=['0']      else:        page_all=str(pages[0])      follow_all=[]      uid_part=[]      print(page_all)
      if(page_all[0]=="0"):        follow_url = 'https://weibo.cn/'+str(id)+'/follow'        login=session.get(url=follow_url,headers=headers_logined,timeout=3)        follow_part=re.findall(r'<td valign="top"><a.*?href="(.*?)">(.*?)</a>',login.text)        follow_all = follow_all+follow_part      else:        #循环取每一页的关注和他的主页，但是这里取不到id，所以还得继续根据主页取
        for page in range(int(page_all)):          page = page+1          follow_url = 'https://weibo.cn/'+str(id)+'/follow?page=' + str(page)          login=session.get(url=follow_url,headers=headers_logined,timeout=3)          #匹配出来结果为关注的用户名昵称，是个二重数组，list里面再有一个list，follow_part[1][0]取主页url，follower[1][1]来取用户名          follow_part=re.findall(r'<td valign="top"><a.*?href="(.*?)">(.*?)</a>',login.text)          follow_all = follow_all+follow_part          time.sleep(3)      #当关注数量大于150个的时候，跳过不计算      #follow_all为所有关注及其对应的主页      print(follow_all)      #根据主页取每一个关注的id，凑成uid_all      name_list=[]      for s in range(len(follow_all)):        if("微博" in follow_all[s][1]):          continue        follower=session.get(follow_all[s][0],headers=headers_logined,timeout=3)        uid = re.findall(r'私信</a>&nbsp;<a href="/(.*?)/info">资料</a>',follower.text)        print(str(s) +'  '+ follow_all[s][1] + ': ')        print(uid)        #这是某一个的总集合        uid_part = uid_part + uid        name_list.append(follow_all[s][1])        #people=people+1        #if(people>50):        #  print("已经爬取前50个关注")        #  break        time.sleep(1.5)      print('\n'+'这是'+id+'__'+name[0]+'的所有关注')      #写入每个文件的一定要是用户名，才好做数据对齐      print(name_list)      file_name=name[0]+'.txt'      with open('./weibo/'+file_name,'w',encoding="utf-8") as f:        f.write(str(name_list))

      #for one in uid_part:      #  url_one='https://weibo.cn/'+one      #  l=session.get(url=url_one,headers=headers_logined)      #  name_one=re.findall(r'<title>(.*?)的微博</title>',l.text)      #  name_list = name_list+name_one
      print('\n该用户的关注名单已经写入文件\n')
      #这层的某一个算完以后加入这一层的总集合)      uid_all=uid_part+uid_all    #print('第'+str(turn)+'层得到的结果：')    #print(uid_all)    #最外层循环结束    print('\n'+'第'+str(turn)+'层结束：')  turn = turn +1  follow_spider(uid_all,turn)

Request URL: https://api.twitter.com/1.1/friends/list.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cursor=-1&user_id=83570815&count=20
Request Method: GETStatus Code: 200 Remote Address: 127.0.0.1:10809Referrer Policy: no-referrer-when-downgradeauthorization: Bearer …….Origin: https://twitter.comReferer: https://twitter.com/Stefsunyanzi/followingSec-Fetch-Mode: corsUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36x-csrf-token: 081e09c5b2c1b13627ee8a9309de6131x-twitter-active-user: yesx-twitter-auth-type: OAuth2Sessionx-twitter-client-language: eninclude_profile_interstitial_type: 1include_blocking: 1include_blocked_by: 1include_followed_by: 1include_want_retweets: 1include_mute_edge: 1include_can_dm: 1include_can_media_tag: 1skip_status: 1cursor: -1user_id: 83570815count: 20

cookie: _ga=******; tfw_exp=0; kdt=w******; remember_checked_on=1; csrf_same_site_set=1; csrf_same_site=1; des_opt_in=Y; fontsLoaded=true; ct0=081e09c5b2c1b13627ee8a9309de6131; _gid=******; lang=en; _sl=1; personalization_id="v******"; guest_id=******; ads_prefs="HBERAAA="; _twitter_sess=******; auth_token=******; rweb_optin=side_no_out; twid=u%3D1203136912770224128; _gat=1

def spider_follower(id_all,name_all,turn):  print("这是第"+str(turn)+"层：")  print(id_all)  print(name_all)  if(turn == 3):    print("达到层数，退出")    exit()  headers_json={  'authorization': 'Bearer …..',  'content-type': 'application/x-www-form-urlencoded',  'Origin': 'https://twitter.com',  'Sec-Fetch-Mode': 'cors',  'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.87 Safari/537.36',  #建议每次用脚本之前，都去浏览器再重新更新一下cookie，以防过期，获取过程很简单，直接看发包就可以了，然后整个脱下来。  'Cookie':'_ga= ,  'x-csrf-token':'97e791a574a3a81468622aefcee61068'}  position=0  id_next_all=[]  follower_name_all=[]  for ids in id_all:    url='https://api.twitter.com/1.1/friends/list.json?include_profile_interstitial_type=1&include_blocking=1&include_blocked_by=1&include_followed_by=1&include_want_retweets=1&include_mute_edge=1&include_can_dm=1&include_can_media_tag=1&skip_status=1&cursor=-1&user_id='+str(ids)+'&count=100'    follower = session.get(url=url,headers=headers_json,proxies=proxies)    follower_json=json.loads(follower.text)    #print(follower_json)    follower_name=[]    id_next=[]    #这个follower是json格式了已经，follower_json["users"]还是一个list    for follower in follower_json["users"]:      #print("screen_name: "+follower["screen_name"])      id_next.append(follower["id_str"])      follower_name.append(follower["name"])    print('\n')    print(ids)    str(name_all[position]).replace(' ','_').replace('?','_').replace('.','_').replace('/','_').replace('\\','_').replace('"','_').replace('<','_').replace('>','_').replace('|','_').replace('\'','_')    filename=str(name_all[position])+'.txt'    print(filename)    print('\n')    try:      with open('./twitter/'+filename,'w',encoding='utf-8') as f:        f.write(str(follower_name))    except:      pass    position=position+1    id_next_all=id_next_all+id_next    follower_name_all = follower_name_all + follower_name  turn = turn +1  print(id_next_all)  print(follower_name_all)  spider_follower(id_next_all,follower_name_all,turn)