每日获取变更的CVE漏洞
2020-03-16 09:00:45 Author: www.freebuf.com(查看原文) 阅读量:485 收藏

查看CVE推送每日更新,做成类似于新闻头条的推送是企业安全从业人员最应该掌控的能力。随着安全体系工作的开展,每位甲方安全从业者从开始的朋友圈接收漏洞信息,到各个平台接收漏洞信息,但无论是三方还是朋友圈,都不能百分之百贴合与及时的自己想要掌控的漏洞信息,也正是基于这点,我开始自己做CVE的推送工作。

首先要爬取CVE,有一个比较方便的网站,内里集成了每天发布或更新的CVE

URL:https://cassandra.cerias.purdue.edu/CVE_changes/today.html

每一个链接都会链接到CVE漏洞详情中

那我们使用python针对CVE进行信息的爬取

headers = {

    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0',

    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

    'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

    'Accept-Encoding':'gzip, deflate',

    'Upgrade-Insecure-Requests':'1',

}

url = "https://cassandra.cerias.purdue.edu/CVE_changes/today.html"

def get_cve_urls():

    '''获取最新的cve漏洞url地址'''

    start_content = 'New entries' # 起始字符串

    end_content = 'Graduations'

    response = requests.get(url, headers=headers, timeout=60)

    response = str(response.text)

    start_index = response.index(start_content)

    if start_index >= 0:

        start_index += len(start_content)

        end_index = response.index(end_content)

        cve_urls_content = response[start_index:end_index]  # 获取网页的指定范围

        soup = BeautifulSoup(cve_urls_content,'lxml')

        cve_url_lists = []     # 存放获取到的cve url

        for u in soup.find_all('a'):

            cve_url = u["href"]

            cve_url_lists.append(cve_url)

#     print(cve_url)

        return cve_url_lists

def get_cve_info():

    '''获取最新cve漏洞信息'''

    print '[*] 最新cve漏洞信息:\n'

    sleep(2)

    cve_urls = get_cve_urls()

    numid = 1

    for cve_url in cve_urls:

        response = requests.get(cve_url,headers=headers,timeout=60)

        response = response.text

        soup = BeautifulSoup(response,'lxml')

        table = soup.find("div",id="GeneratedTable").find("table")    # 获取table标签内容

        cve_id = table.find_all("tr")[1].find("td",nowrap="nowrap").find("h2").string   # cve id

        cve_description = table.find_all("tr")[3].find("td").string       # cve 介绍

其中会有一部分英文的CVE介绍会存在特殊字符,比如单引号,这时我们需要将单引号做处理后才能输出

    if str(cve_description).find('\'') != -1:

            cve_description = str(cve_description).replace('\'', '')

            print('替换特殊字符处理--\'')

            print(str(cve_description))

CVE介绍为英文,如果想翻译安装trans插件,详细请自行百度

由于每天新增的CVE过多,可以添加自己关注的组件漏洞,关注的漏洞才发送

由于CVE官方并没有漏洞等级的介绍,可以将此CVE放到NVD中获取漏洞风险等级

   base_url = 'https://nvd.nist.gov/vuln/detail/'+cve_id

            base_score = requests.get(base_url,headers=headers,timeout=60)

            response_score = base_score.text

            soup_score = BeautifulSoup(response_score,'lxml')

    soup_score_div = soup_score.find("div",id="p_lt_WebPartZone1_zoneCenter_pageplaceholder_p_lt_WebPartZone1_zoneCenter_VulnerabilityDetail_VulnFormView_Vuln3CvssPanel")

    soup_score_tag = soup_score_div.find_all(id=re.compile("p_lt_WebPartZone1_zoneCenter_pageplaceholder_p_lt_WebPartZone1_zoneCenter_VulnerabilityDetail_VulnFormView_Cvss3NistCalculatorAnchor*"))[0].string

    print(soup_score_tag)

            print("[+] cve漏洞等级:"+soup_score_tag)

如此基本集成了漏洞推送的各个组件

整体代码:

from time import sleep

import requests

from bs4 import BeautifulSoup

import re

import smtplib

from email.mime.text import MIMEText

from email.header import Header

import datetime

import os

import sys

cvelist=[]

cvelist.append('<H1><b>New vulnerability ')

if sys.getdefaultencoding() != 'utf-8':

    reload(sys)

    sys.setdefaultencoding('utf-8')

now_time = datetime.datetime.today().strftime('%Y,%m,%d')

yesterday_time = datetime.datetime.today()+datetime.timedelta(-1)

yesterday = yesterday_time.strftime('%Y.%m.%d')

now_year = yesterday_time.strftime('%Y')

print(yesterday)

cvelist.append(now_time)

cvelist.append('</b></H1><p></p><p></p><p></p>')

component_lists = ['tomcat','nginx','apache','kibana','elastic','logstash','jackson','fastjson','windows','win10','win7','linux','centos','ssh','kernel','jenkins','zabbix','grafana','kubernetes','docker']

headers = {

    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0',

    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

    'Accept-Language':'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

    'Accept-Encoding':'gzip, deflate',

    'Upgrade-Insecure-Requests':'1',

}

url = "https://cassandra.cerias.purdue.edu/CVE_changes/today.html"

def get_cve_urls():

    '''获取最新的cve漏洞url地址'''

    start_content = 'New entries' # 起始字符串

    end_content = 'Graduations'

    response = requests.get(url, headers=headers, timeout=60)

    response = str(response.text)

    start_index = response.index(start_content)

    if start_index >= 0:

        start_index += len(start_content)

        end_index = response.index(end_content)

        cve_urls_content = response[start_index:end_index]  # 获取网页的指定范围

        soup = BeautifulSoup(cve_urls_content,'lxml')

        cve_url_lists = []     # 存放获取到的cve url

        for u in soup.find_all('a'):

            cve_url = u["href"]

            cve_url_lists.append(cve_url)

#     print(cve_url)

        return cve_url_lists

def get_cve_info():

    '''获取最新cve漏洞信息'''

    print '[*] 最新cve漏洞信息:\n'

    sleep(2)

    cve_urls = get_cve_urls()

    numid = 1

    for cve_url in cve_urls:

        response = requests.get(cve_url,headers=headers,timeout=60)

        response = response.text

        soup = BeautifulSoup(response,'lxml')

        table = soup.find("div",id="GeneratedTable").find("table")    # 获取table标签内容

        cve_id = table.find_all("tr")[1].find("td",nowrap="nowrap").find("h2").string   # cve id

        cve_description = table.find_all("tr")[3].find("td").string       # cve 介绍

        print "[+] cve漏洞编号:",cve_id

        if str(cve_description).find('\'') != -1:

            cve_description = str(cve_description).replace('\'', '')

            print('替换特殊字符处理--\'')

            print(str(cve_description))

        if any(component in str(cve_description) for component in component_lists):

            oscve = "trans en:zh-CN '"+str(cve_description)+"'|awk 'NR==3 {print $0}'"

            oscve_zh = os.popen(oscve).read()

            cvetitle = '<H1><b><span style="background-color: rgb(255, 153, 0);">'+str(numid)+'.CVE</span></b></H1>'

            cvelist.append(cvetitle)

            numid=numid+1

            cvelist.append('</p><p>vulnerability URL:')

            cvelist.append(cve_url)

            cvelist.append('</p><p>cve id:')

            cvelist.append(cve_id)

            cvelist.append('</p><p>vulnerability introduction</p><p>')

            cvelist.append(str(cve_description))

            cvelist.append('</p><p>译文:')

            cvelist.append(oscve_zh)

            base_url = 'https://nvd.nist.gov/vuln/detail/'+cve_id

            base_score = requests.get(base_url,headers=headers,timeout=60)

            response_score = base_score.text

            soup_score = BeautifulSoup(response_score,'lxml')

    soup_score_div = soup_score.find("div",id="p_lt_WebPartZone1_zoneCenter_pageplaceholder_p_lt_WebPartZone1_zoneCenter_VulnerabilityDetail_VulnFormView_Vuln3CvssPanel")

    soup_score_tag = soup_score_div.find_all(id=re.compile("p_lt_WebPartZone1_zoneCenter_pageplaceholder_p_lt_WebPartZone1_zoneCenter_VulnerabilityDetail_VulnFormView_Cvss3NistCalculatorAnchor*"))[0].string

    print(soup_score_tag)

            print("[+] cve漏洞等级:"+soup_score_tag)

            cvelist.append('</p><p>cve漏洞等级:')

            cvelist.append(soup_score_tag)

            cvelist.append('</p><p></p>')

        else:

            print('No Date')

    mail_host=" "  #设置服务器

    mail_user=" "    #用户名

    mail_pass=" "   #口令

    sender = ' '  #发件人

    receivers = [' ']  #收件人

    mail_msg=''.join(cvelist)

    message = MIMEText(mail_msg, 'html', 'utf-8')

    message['From'] = "{}".format(sender)

    message['To'] = ",".join(receivers)

    subject = yesterday+'CVE收录新增漏洞'

    message['Subject'] = Header(subject, 'utf-8')

    try:

  smtpObj = smtplib.SMTP_SSL(mail_host, 465)

  smtpObj.login(mail_user, mail_pass)

  smtpObj.sendmail(sender, receivers, message.as_string())

  print "邮件发送成功"

    except smtplib.SMTPException:

  print "Error: 无法发送邮件"

def main():

    get_cve_info()

if __name__ == "__main__":

    main()

请根据自己的情况填写邮箱,由于爬取CVE的网站是每天17:02更新漏洞,所以每天早上获取漏洞的小伙伴记得要采用yesterday变量,每天晚上获取漏洞的小伙伴采用today即可。

效果如下:

*本文原创作者:煜阳yuyang,本文属于FreeBuf原创奖励计划,未经许可禁止转载


文章来源: https://www.freebuf.com/articles/es/228571.html
如有侵权请联系:admin#unsafe.sh