diff options
| -rw-r--r-- | lib/tasks/cve.rake | 422 |
1 files changed, 251 insertions, 171 deletions
diff --git a/lib/tasks/cve.rake b/lib/tasks/cve.rake index dbe06cf..4fccf47 100644 --- a/lib/tasks/cve.rake +++ b/lib/tasks/cve.rake @@ -1,5 +1,6 @@ # ===GLSAMaker v2 # Copyright (C) 2010–15 Alex Legler <a3li@gentoo.org> +# Copyright (C) 2020 Max Magorsch <arzano@gentoo.org> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -7,8 +8,7 @@ # (at your option) any later version. # # For more information, see the LICENSE file. - -libs = %w( nokogiri zlib stringio ) +libs = %w( json nokogiri zlib stringio ) libs << File.join(File.dirname(__FILE__), '..', 'bugzilla') libs << File.join(File.dirname(__FILE__), '..', 'glsamaker') libs << File.join(File.dirname(__FILE__), 'utils') @@ -18,7 +18,7 @@ libs.each { |lib| require lib } TMPDIR = File.join(File.dirname(__FILE__), '..', '..', 'tmp') # What year are the first CVEs from? YEAR = (ENV['START_YEAR'] || 2004).to_i -BASEURL = 'https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz' +BASEURL = 'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%s.json.gz' DEBUG = ENV.key? 'DEBUG' VERBOSE = (ENV.key?('VERBOSE') || DEBUG) @@ -27,6 +27,23 @@ QUIET = ENV.key? 'QUIET' fail "I can't be quiet and verbose at the same time..." if QUIET && VERBOSE namespace :cve do + + desc 'Destroy all CVEs and CPEs' + task destroy_all: [:environment, 'db:load_config'] do + CveReference.destroy_all + Cve.destroy_all + Cpe.destroy_all + end + + + desc 'Print all CVEs' + task print_all: [:environment, 'db:load_config'] do + Cve.find_each do |cve| + puts cve.cve_id + ",\"" + cve.summary + "\"," + (cve.cvss || "") + "," + cve.state + "," + cve.published_at.to_formatted_s(:iso8601) + "," + cve.last_changed_at.to_formatted_s(:iso8601) + "," + cve.created_at.to_formatted_s(:iso8601) + "," + cve.updated_at.to_formatted_s(:iso8601) + end + end + + desc 'Full CVE data import' task full_import: [:environment, 'db:load_config'] do start_ts = Time.now @@ -34,43 +51,16 @@ namespace :cve do (YEAR..Date.today.year).each do |year| info 'Processing CVEs from '.bold + year.to_s.purple - xmldata = status 'Downloading' do + jsondata = status 'Downloading' do gunzip_str Glsamaker::HTTP.get(BASEURL % year) end - xml = status 'Loading XML' do - Nokogiri::XML(xmldata) + json = status 'Loading JSON' do + JSON.parse(jsondata) end - namespace = { 'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0' } - processed_cves = 0 - cpe_cache = {} - - cves = xml.root.xpath('cve:entry', namespace) - info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)" - - cves.each do |cve| - unless VERBOSE || QUIET - - if processed_cves % 500 == 0 - print '.'.purple - else - print '.' if processed_cves % 100 == 0 - end - end - - puts cve['id'].to_s.purple if VERBOSE - - begin - create_cve(cve) - rescue ActiveRecord::StatementInvalid => e - # Ignore dupes, ONLY do that for the full db update! - raise e unless e.message =~ /Duplicate entry/ - end - - processed_cves += 1 - STDOUT.flush - end + cves = json["CVE_Items"] + create_cves(cves) puts end @@ -79,139 +69,273 @@ namespace :cve do info "(#{Time.now - start_ts} seconds)" end - desc 'Incremental CVE data update' + + desc 'Incrementally update last CVEs' task update: :environment do - start_ts = Time.now + info 'Running incremental CVE data update...' - xmldata = status 'Downloading' do + jsondata = status 'Downloading' do gunzip_str Glsamaker::HTTP.get(BASEURL % 'modified') end - xml = status 'Loading XML' do - Nokogiri::XML(xmldata) + json = status 'Loading JSON' do + JSON.parse(jsondata) end - namespace = { 'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0' } - processed_cves = created_cves = updated_cves = 0 - cpe_cache = {} + cves = json["CVE_Items"] + update_cves(cves) + end + + + desc 'Update all CVEs' + task update_all: :environment do - cves = xml.root.xpath('cve:entry', namespace) - info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)" + info 'Running complete CVE data update...' + + (YEAR..Date.today.year).each do |year| + info 'Processing CVEs from '.bold + year.to_s.purple + + jsondata = status 'Downloading' do + gunzip_str Glsamaker::HTTP.get(BASEURL % year) + end + + json = status 'Loading JSON' do + JSON.parse(jsondata) + end - cves.each do |cve| - unless VERBOSE || QUIET - if processed_cves % 500 == 0 - print '.'.purple - else - print '.' if processed_cves % 100 == 0 + cves = json["CVE_Items"] + update_cves(cves) + end + end + +end + + +# +# Update the given cve +# +def create_cve(cve) + summary = cve.dig('cve', 'description', 'description_data', 0, 'value') + _cve = Cve.create( + :cve_id => cve.dig('cve', 'CVE_data_meta', 'ID'), + :summary => summary, + :cvss => cve.dig('impact', 'baseMetricV2', 'cvssV2', 'vectorString'), + :published_at => DateTime.parse(cve['publishedDate']), + :last_changed_at => DateTime.parse(cve['lastModifiedDate']), + :state => (summary =~ /^\*\* REJECT \*\*/ ? 'REJECTED' : 'NEW') + ) + + if cve.dig('references', 'reference_data') + cve.dig('references', 'reference_data').each do |ref| + CveReference.create( + :cve => _cve, + :source => ref['refsource'], + :title => ref['name'], + :uri => ref['url'] + ) + end + end + + cve.dig('configurations', 'nodes').each do |node| + if node.key?('cpe_match') + node['cpe_match'].each do |cpe_match| + cpe_str = cpe_match['cpe23Uri'] + + cpe = Cpe.where(:cpe => cpe_str).first + cpe ||= Cpe.create(:cpe => cpe_str) + + _cve.cpes << cpe + end + elsif node.key?('children') + node['children'].each do |child| + if child.key?('cpe_match') + child['cpe_match'].each do |cpe_match| + cpe_str = cpe_match['cpe23Uri'] + + cpe = Cpe.where(:cpe => cpe_str).first + cpe ||= Cpe.create(:cpe => cpe_str) + + _cve.cpes << cpe + end end end + end + end +end - puts cve['id'].to_s.purple if VERBOSE - c = Cve.find_by_cve_id cve['id'] +# +# Create the all given cves +# +def create_cves(cves) + processed_cves = 0 + info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)" + + cves.each do |cve| + unless VERBOSE || QUIET - if c.nil? - debug 'Creating CVE.' - create_cve(cve) - created_cves += 1 + if processed_cves % 500 == 0 + print '.'.purple else - last_changed_at = Time.parse(cve.xpath('vuln:last-modified-datetime').first.content).utc - db_lca = c.last_changed_at - - if last_changed_at.to_i > c.last_changed_at.to_i - debug 'Updating CVE. Timestamp changed.' - summary = cve.xpath('vuln:summary').first.content - c.attributes = { - cve_id: cve['id'], + print '.' if processed_cves % 100 == 0 + end + end + + puts cve.dig('cve', 'CVE_data_meta', 'ID').to_s.purple if VERBOSE + + begin + create_cve(cve) + rescue ActiveRecord::StatementInvalid => e + # Ignore dupes, ONLY do that for the full db update! + raise e unless e.message =~ /Duplicate entry/ + end + + processed_cves += 1 + STDOUT.flush + end +end + + +# +# Update the all given cves +# +def update_cves(cves) + start_ts = Time.now + processed_cves = created_cves = updated_cves = 0 + + info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)" + + cves.each do |cve| + unless VERBOSE || QUIET + if processed_cves % 500 == 0 + print '.'.purple + else + print '.' if processed_cves % 100 == 0 + end + end + + puts cve.dig('cve', 'CVE_data_meta', 'ID').to_s.purple if VERBOSE + + c = Cve.find_by_cve_id cve.dig('cve', 'CVE_data_meta', 'ID') + + if c.nil? + debug 'Creating CVE.' + create_cve(cve) + created_cves += 1 + else + last_changed_at = Time.parse(cve['lastModifiedDate']).utc + + if last_changed_at.to_i > c.last_changed_at.to_i + debug 'Updating CVE. Timestamp changed.' + summary = cve.dig('cve', 'description', 'description_data', 0, 'value') + c.attributes = { + cve_id: cve.dig('cve','CVE_data_meta','ID'), summary: summary, - cvss: cvss_xml2str(cve.xpath('vuln:cvss')), - published_at: DateTime.parse(cve.xpath('vuln:published-datetime').first.content), - last_changed_at: DateTime.parse(cve.xpath('vuln:last-modified-datetime').first.content) - } + cvss: cve.dig('impact', 'baseMetricV2', 'cvssV2', 'vectorString'), + published_at: DateTime.parse(cve['publishedDate']), + last_changed_at: DateTime.parse(cve['lastModifiedDate']), + } - c.state = 'REJECTED' if summary =~ /^\*\* REJECT \*\*/ - c.save! + c.state = 'REJECTED' if summary =~ /^\*\* REJECT \*\*/ + c.save! - db_references = [] - xml_references = [] - cve.xpath('vuln:references').each do |ref| + db_references = [] + xml_references = [] + + if cve.dig 'references', 'reference_data' + cve.dig('references', 'reference_data').each do |ref| xml_references << [ - ref.xpath('vuln:source').first.content, - ref.xpath('vuln:reference').first.content, - ref.xpath('vuln:reference').first['href'] + :source => ref['refsource'], + :title => ref['name'], + :uri => ref['url'] ] end + end - c.references.each do |ref| - db_references << [ref.source, ref.title, ref.uri] - end + c.references.each do |ref| + db_references << [ref.source, ref.title, ref.uri] + end - rem = db_references - xml_references - debug "Removing references: #{rem.inspect}" + rem = db_references - xml_references + debug "Removing references: #{rem.inspect}" - rem.each do |item| - ref = c.references.where(['source = ? AND title = ? AND uri = ?', *item]).first - debug ref - c.references.delete(ref) - ref.destroy - end + rem.each do |item| + ref = c.references.where(['source = ? AND title = ? AND uri = ?', *item]).first + debug ref + c.references.delete(ref) + ref.destroy + end - add = xml_references - db_references - debug "Ading references: #{add.inspect}" + add = xml_references - db_references + debug "Ading references: #{add.inspect}" - add.each do |item| - c.references.create( + add.each do |item| + c.references.create( source: item[0], title: item[1], uri: item[2] - ) - end - - db_cpes = [] - xml_cpes = [] - cve.xpath('vuln:vulnerable-software-list/vuln:product').each do |prod| - xml_cpes << prod.content - end + ) + end - c.cpes.each do |prod| - db_cpes << prod.cpe + db_cpes = [] + xml_cpes = [] + + cve.dig('configurations', 'nodes').each do |node| + if node.key?('cpe_match') + node['cpe_match'].each do |cpe_match| + xml_cpes << cpe_match['cpe23Uri'] + end + elsif node.key?('children') + node['children'].each do |child| + if child.key?('cpe_match') + child['cpe_match'].each do |cpe_match| + xml_cpes << cpe_match['cpe23Uri'] + end + end + end end + end - rem = db_cpes - xml_cpes - debug "Removing CPEs: #{rem.inspect}" + c.cpes.each do |prod| + db_cpes << prod.cpe + end - rem.each do |item| - c.cpes.delete(Cpe.find_by_cpe(item)) - end + rem = db_cpes - xml_cpes + debug "Removing CPEs: #{rem.inspect}" - add = xml_cpes - db_cpes - debug "Ading CPEs: #{add.inspect}" + rem.each do |item| + c.cpes.delete(Cpe.find_by_cpe(item)) + end - add.each do |item| - cpe = Cpe.where(cpe: item).first - cpe ||= Cpe.create(cpe: item) + add = xml_cpes - db_cpes + debug "Ading CPEs: #{add.inspect}" - c.cpes << cpe - end + add.each do |item| + cpe = Cpe.where(cpe: item).first + cpe ||= Cpe.create(cpe: item) - c.save! - updated_cves += 1 + c.cpes << cpe end - end - processed_cves += 1 - STDOUT.flush + c.save! + updated_cves += 1 + end end - info '' - - info "(#{Time.now - start_ts} seconds, #{created_cves} new CVE entries, #{updated_cves} updated CVE entries)" + processed_cves += 1 + STDOUT.flush end + + info '' + + info "(#{Time.now - start_ts} seconds, #{created_cves} new CVE entries, #{updated_cves} updated CVE entries)" end + +# # Run something, and display a status info around it +# def status(message) fail ArgumentError, 'I want a block :(' unless block_given? @@ -223,63 +347,19 @@ def status(message) stuff end + def debug(msg) $stderr.puts msg if DEBUG end + def info(msg) puts msg unless QUIET end -# 7.2/AV:L/AC:L/Au:N/C:C/I:C/A:C -def cvss_xml2str(data) - def get_content(x, y) - x.xpath(y).first.content - end - - return nil if data.size == 0 - - str = "#{get_content(data, 'cvss:base_metrics/cvss:score')}/" - str += "AV:#{get_content(data, 'cvss:base_metrics/cvss:access-vector')[0, 1]}/" - str += "AC:#{get_content(data, 'cvss:base_metrics/cvss:access-complexity')[0, 1]}/" - str += "Au:#{get_content(data, 'cvss:base_metrics/cvss:authentication')[0, 1]}/" - str += "C:#{get_content(data, 'cvss:base_metrics/cvss:confidentiality-impact')[0, 1]}/" - str += "I:#{get_content(data, 'cvss:base_metrics/cvss:integrity-impact')[0, 1]}/" - str += "A:#{get_content(data, 'cvss:base_metrics/cvss:availability-impact')[0, 1]}" - - str -end - -def create_cve(cve) - summary = cve.xpath('vuln:summary').first.content - _cve = Cve.create( - :cve_id => cve['id'], - :summary => summary, - :cvss => cvss_xml2str(cve.xpath('vuln:cvss')), - :published_at => DateTime.parse(cve.xpath('vuln:published-datetime').first.content), - :last_changed_at => DateTime.parse(cve.xpath('vuln:last-modified-datetime').first.content), - :state => (summary =~ /^\*\* REJECT \*\*/ ? 'REJECTED' : 'NEW') - ) - - cve.xpath('vuln:references').each do |ref| - CveReference.create( - :cve => _cve, - :source => ref.xpath('vuln:source').first.content, - :title => ref.xpath('vuln:reference').first.content, - :uri => ref.xpath('vuln:reference').first['href'] - ) - end - - cve.xpath('vuln:vulnerable-software-list/vuln:product').each do |prod| - cpe_str = prod.content - - cpe = Cpe.where(:cpe => cpe_str).first - cpe ||= Cpe.create(:cpe => cpe_str) - - _cve.cpes << cpe - end -end - +# +# unzip the given string +# def gunzip_str(str) Zlib::GzipReader.new(StringIO.new(str)).read end |