json fallback (#16)

* orjson fallback

* orjson fallback

* orjson fallback

Author: prabhu

.github/workflows/pythonapp.yml

@@ -5,11 +5,11 @@ on: [push]
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.8]
+        os: [ubuntu-20.04, macos-latest, windows-latest]
+        python-version: [3.8, 3.9]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python 3.7
@@ -31,6 +31,10 @@ jobs:
     - name: Test with pytest
       run: |
         pytest --cov=vdb test
+    - name: Test without orjson
+      run: |
+        pip uninstall -y orjson
+        pytest --cov=vdb test
     - name: Self sast-scan
       uses: AppThreat/sast-scan-action@master
       with:

.github/workflows/pythonpublish.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   deploy:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
       - uses: actions/checkout@v2

requirements.txt

@@ -2,5 +2,5 @@ requests
 appdirs
 tabulate
 msgpack
-orjson==3.3.1
+orjson
 semver

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="appthreat-vulnerability-db",
-    version="1.5.2",
+    version="1.6.0",
     author="Team AppThreat",
     author_email="cloud@appthreat.com",
     description="AppThreat's vulnerability database and package search library with a built-in file based storage. CVE, GitHub, npm are the primary sources of vulnerabilities.",
@@ -14,7 +14,7 @@
     long_description_content_type="text/markdown",
     url="https://github.com/appthreat/vulnerability-db",
     packages=setuptools.find_packages(),
-    install_requires=["requests", "appdirs", "tabulate", "msgpack", "orjson==3.3.1", "semver"],
+    install_requires=["requests", "appdirs", "tabulate", "msgpack", "orjson", "semver"],
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",

test/test_db.py

@@ -96,7 +96,11 @@ def test_gha_search_slow(test_db, test_gha_data):
     for d in all_data:
         version = d["details"]["max_affected_version_including"]
         if version and version != "*":
-            res = db.pkg_search(table, d["details"]["package"], version,)
+            res = db.pkg_search(
+                table,
+                d["details"]["package"],
+                version,
+            )
             assert len(res)
             assert res[0].to_dict()["package_issue"]
 
@@ -113,7 +117,12 @@ def test_gha_vendor_search(test_db, test_gha_data):
         vendor, _, _ = parse_cpe(d["details"]["cpe_uri"])
         version = d["details"]["max_affected_version_including"]
         if version and version != "*":
-            res = db.vendor_pkg_search(table, vendor, d["details"]["package"], version,)
+            res = db.vendor_pkg_search(
+                table,
+                vendor,
+                d["details"]["package"],
+                version,
+            )
             assert len(res)
             assert res[0].to_dict()["package_issue"]
 

vdb/cli.py

@@ -94,8 +94,7 @@ def build_args():
 
 
 def print_results(results):
-    """Pretty print report summary
-    """
+    """Pretty print report summary"""
     table = []
     added_list = []
     headers = [

vdb/lib/__init__.py

@@ -80,8 +80,7 @@ def convert_time(time_str):
 
 
 class Vulnerability(object):
-    """Vulnerability
-    """
+    """Vulnerability"""
 
     def __init__(
         self,
@@ -124,8 +123,7 @@ def __repr__(self):
 
 
 class VulnerabilityDetail(object):
-    """Vulnerability detail class
-    """
+    """Vulnerability detail class"""
 
     def __init__(
         self,
@@ -202,8 +200,7 @@ def from_dict(detail):
 
 
 class PackageIssue(object):
-    """Package issue class
-    """
+    """Package issue class"""
 
     def __init__(
         self,
@@ -253,8 +250,7 @@ def __str__(self):
 
 
 class CvssV3(object):
-    """CVSS v3 representation
-    """
+    """CVSS v3 representation"""
 
     base_score: float
     exploitability_score: float
@@ -370,8 +366,7 @@ def __str__(self):
 
 
 class VulnerabilityOccurrence:
-    """Class to represent an occurrence of a vulnerability
-    """
+    """Class to represent an occurrence of a vulnerability"""
 
     id: str
     problem_type: str
@@ -409,8 +404,7 @@ def __init__(
         self.effective_severity = effective_severity
 
     def to_dict(self):
-        """Convert the object to dict
-        """
+        """Convert the object to dict"""
         return {
             "id": self.id,
             "problem_type": self.problem_type,

vdb/lib/db.py

@@ -159,8 +159,7 @@ def _key_func(data, match_list):
 
 
 def bulk_index_search(pkg_list):
-    """
-    """
+    """"""
     ret_list = set()
     for pkg in pkg_list:
         version_list = None

vdb/lib/gha.py

@@ -12,7 +12,15 @@
 import os
 import re
 
-import orjson
+try:
+    import orjson
+
+    ORJSON_AVAILABLE = True
+except ImportError:
+    import json
+
+    ORJSON_AVAILABLE = False
+
 import requests
 
 from vdb.lib import config as config
@@ -27,10 +35,11 @@
 api_token = os.environ.get("GITHUB_TOKEN")
 headers = {"Authorization": "token %s" % api_token}
 
+json_lib = orjson if ORJSON_AVAILABLE else json
+
 
 def get_query(type="recent"):
-    """
-    """
+    """"""
     extra_args = ""
     if type == "recent" or not type:
         extra_args = "first: 100"
@@ -85,12 +94,10 @@ def get_query(type="recent"):
 
 
 class GitHubSource(NvdSource):
-    """GitHub CVE source
-    """
+    """GitHub CVE source"""
 
     def download_all(self, local_store=True):
-        """Download all historic cve data
-        """
+        """Download all historic cve data"""
         data_list = []
         lastId = None
         for y in range(0, int(config.gha_pages_count)):
@@ -104,16 +111,14 @@ def download_all(self, local_store=True):
         return data_list
 
     def download_recent(self, local_store=True):
-        """Method which downloads the recent CVE
-        """
+        """Method which downloads the recent CVE"""
         data, page_info = self.fetch("recent")
         if data and local_store:
             self.store(data)
         return data
 
     def fetch(self, type):
-        """Private method to fetch the advisory data via GraphQL api
-        """
+        """Private method to fetch the advisory data via GraphQL api"""
         LOG.debug(
             "Download GitHub advisory from {} with cursor {}".format(
                 config.gha_url, type
@@ -171,8 +176,7 @@ def get_version_range(self, version_str):
         )
 
     def convert(self, cve_data):
-        """Convert the GitHub advisory data into Vulnerability objects
-        """
+        """Convert the GitHub advisory data into Vulnerability objects"""
         ret_data = []
         if cve_data.get("errors"):
             return ret_data, None
@@ -195,6 +199,9 @@ def convert(self, cve_data):
             if not cve_id:
                 cve_id = cve["ghsaId"]
                 assigner = "@github"
+            references = json_lib.dumps(references)
+            if isinstance(references, bytes):
+                references = references.decode("utf-8", "ignore")
             for p in cve["vulnerabilities"]["nodes"]:
                 vendor = p["package"]["ecosystem"]
                 product = p["package"]["name"]
@@ -234,11 +241,12 @@ def convert(self, cve_data):
             """.format(
                     cve.get("summary"), cve.get("description")
                 )
+
                 tdata = config.CVE_TPL % dict(
                     cve_id=cve_id,
                     cwe_id="UNKNOWN",
                     assigner=assigner,
-                    references=orjson.dumps(references).decode("utf-8"),
+                    references=references,
                     description="",
                     vectorString=vectorString,
                     vendor=vendor.lower(),
@@ -260,7 +268,7 @@ def convert(self, cve_data):
                     lastModifiedDate=cve["updatedAt"],
                 )
                 try:
-                    tdata_json = orjson.loads(tdata)
+                    tdata_json = json_lib.loads(tdata)
                     vuln = NvdSource.convert_vuln(tdata_json)
                     vuln.description = description
                     ret_data.append(vuln)

vdb/lib/npm.py

@@ -5,7 +5,15 @@
 """
 import logging
 
-import orjson
+try:
+    import orjson
+
+    ORJSON_AVAILABLE = True
+except ImportError:
+    import json
+
+    ORJSON_AVAILABLE = False
+
 import requests
 
 from vdb.lib import config as config
@@ -23,6 +31,9 @@
 )
 LOG = logging.getLogger(__name__)
 
+json_lib = orjson if ORJSON_AVAILABLE else json
+import traceback
+
 
 class NpmSource(NvdSource):
     """
@@ -67,8 +78,7 @@ def fetch(self, payload):
         return self.convert(json_data)
 
     def download_recent(self, local_store=True):
-        """Method which downloads the recent CVE
-        """
+        """Method which downloads the recent CVE"""
         url = config.npm_advisories_url + "?perPage=100&page=1"
         r = requests.get(url=url)
         if r.ok:
@@ -80,8 +90,7 @@ def download_recent(self, local_store=True):
         return None
 
     def download_all(self, local_store=True):
-        """Download all historic cve data
-        """
+        """Download all historic cve data"""
         data_list = []
         url = config.npm_advisories_url + "?perPage=100&page=1"
         for y in range(0, int(config.npm_pages_count)):
@@ -185,6 +194,9 @@ def to_vuln(self, v, ret_data):
             )
             if v.get("references"):
                 references = convert_md_references(v.get("references"))
+            references = json_lib.dumps(references)
+            if isinstance(references, bytes):
+                references = references.decode("utf-8", "ignore")
             severity = v.get("severity")
             vendor = "npm"
             product = v["module_name"]
@@ -223,11 +235,12 @@ def to_vuln(self, v, ret_data):
                     "version_end_excluding", ""
                 )
                 description = fix_text(description)
+
                 tdata = config.CVE_TPL % dict(
                     cve_id=cve_id,
                     cwe_id=cwe_id,
                     assigner=assigner,
-                    references=orjson.dumps(references).decode("utf-8"),
+                    references=references,
                     description="",
                     vectorString=vectorString,
                     vendor=vendor,
@@ -249,10 +262,12 @@ def to_vuln(self, v, ret_data):
                     lastModifiedDate=lastModifiedDate,
                 )
                 try:
-                    vuln = NvdSource.convert_vuln(orjson.loads(tdata))
+                    vuln = NvdSource.convert_vuln(json_lib.loads(tdata))
                     vuln.description = description
                     ret_data.append(vuln)
-                except Exception:
-                    pass
+                except Exception as e:
+                    LOG.debug(e)
+                    traceback.print_exc()
+                    print(tdata)
                 vr = vr + 1
         return ret_data