Skip to content

Latest commit

 

History

History
439 lines (357 loc) · 13 KB

README.md

File metadata and controls

439 lines (357 loc) · 13 KB
Raw

GSD Analysis

The primary goal of this analysis is to understand the data sources, structure, and counts of the Global Security Database (GSD).

Replication

Clone both this repository and the gsd-database repository.

Checkout the gsd-database commit to (gsd timestamp: 2022-05-20T05:00:05:000) replicate below results:

:~/gsd-database$ git checkout d8ce33d48de2f00130e821e9828c3e04b9a4b520

From within the gsd-analysis cloned repo run the following:

python3 -m venv venv
source venv/bin/activate
pip3 install -r requirements.txt
python3 gsd_analysis.py /path_to_gsd-database_repo/
deactivate

Expected Runtime: 1 hour 20 mins

Expected output:

$ tree ./data --dirsfirst
./data
├── figs
│   └── gsd_total_count.png #Figure of updated counts for each GSD object
├── schemas
│   ├── gsd_complete_schema.json #Complete schema of all GSD entries
│   ├── schema_cisa.json #CISA object schema
│   ├── schema_cve_org.json #CVE.org object schema
│   ├── schema_gitlab.json #GitLab object schema
│   ├── schema_gsd_object.json #GSD object schema
│   ├── schema_nvd.json #NVD object schema
│   └── schema_osv.json #OSV object schema
├── gsd_counts_20220520.csv #General count information of all GSD entries
└── gsd_entries_20220520.csv #List of all GSD entries (path, year, group_id, gsd, api)

TODO: Describe gsd_counts

GSD Complete Schema

The schema was generated using the GenSON package. Each GSD JSON was opened and added to the GenSON schema object to create an overall schema for all potential entries within GSD.

Below is a condensed schema after loading all GSD entries.

{
    "GSD": {"type":  "object"},
    "OSV": {"type":  "object"},
    "namespaces": {
        "properties": {
            "cisa.gov": {"type":  "object"},
            "cve.org": {"type":  "object"},
            "gitlab.com": {"type":  "object"},
            "nvd.nist.gov": {"type":  "object"},
            "github.com/kurtseifried:582211": {"type":  "object"}
        }
    },
    "overlay": {
        "properties": {
            "cve.org": {"type":  "object"}
        }
    }
}

Complete schema: ./data/schemas/gsd_complete_schema.json

Data Sources

From the above schema a set of data sources/keys can be identified:

alt text

GSD:

  • Ideally, every entry should have a GSD identifier.
GSD Schema:
{
    "affected": {"type":  "array"},
    "affected_component": {"type":  "string"},
    "affects": {"type":  "object"},
    "alias": {"type":  "string"},
    "attack_vector": {"type":  "string"},
    "credit": {"type":  "string"},
    "data_format": {"type":  "string"},
    "data_type": {"type":  "string"},
    "data_version": {"type":  "string"},
    "description": {"type":  "string"},
    "details": {"type":  "string"},
    "duplicate": {"type":  "string"},
    "extended_references": {"type":  "array"},
    "id": {"type":  "string"},
    "impact": {"type":  "object"},
    "modified":{"type":  "string"},
    "notes": {"type":  "string"},
    "product_name": {"type": "object"},
    "product_version": {"type":  "string"},
    "published": {"type":  "string"},
    "references": {"type":  "array"},
    "reporter": {"type":  "string"},
    "reporter_id": {"type":  "integer"},
    "summary": {"type":  "string"},
    "vendor_name": {"type":  "string"},
    "vulnerability_type": {"type": "object"},
    "withdrawn": {"type":  "string"}
}

Complete GSD object schema: ./data/schemas/schema_gsd_object.json

GSD Outliers:
GSD entries with the same CVE Alias

OSV:

The actual OSV format: https://ossf.github.io/osv-schema/

Examples from GSD with an OSV object:

OSV Schema
{
    "properties": {
        "affected": {"type":  "array"},
        "alias": {"type":  "string"},
        "details": {"type":  "string"}, 
        "id": {"type":  "string"},
        "modified": {"type":  "string"},
        "published": {"type":  "string"},
        "references": {"type":  "array"},
        "summary": {"type":  "string"}
    },
    "required": [
        "affected",
        "details",
        "id",
        "modified",
        "published",
        "summary"
    ],
    "type": "object"
}

Complete OSV object schema: ./data/schemas/schema_osv.json

cisa.gov:

Examples:

cisa.gov schema:
{
    "properties": {
        "cveID": {"type": "string"},
        "dateAdded": {"type": "string"},
        "dueDate": {"type": "string"},
        "product": {"type": "string"},
        "requiredAction": {"type": "string"},
        "shortDescription": {"type": "string"},
        "vendorProject": {"type": "string"},
        "vulnerabilityName": {"type": "string"}
    },
    "required": [
        "cveID",
        "dateAdded",
        "dueDate",
        "product",
        "requiredAction",
        "shortDescription",
        "vendorProject",
        "vulnerabilityName"
    ],
    "type": "object"
}

Complete cisa.gov object schema: ./data/schemas/schema_cisa.json

cve.org:

Examples:

cve.org schema:
{
    "properties": {
        "CVE_data_meta": {"type":  "object"},
        "affects": {"type":  "object"},
        "configuration": {"type":  "object"},
        "credit": {"type":  "object"},
        "cve_id": {"type":  "string"},
        "data_format": {"type":  "string"},
        "data_type": {"type":  "string"},
        "data_version": {"type":  "string"},
        "description": {"type":  "object"},
        "discoverer": {"type":  "string"},
        "exploit": {"type":  "object"},
        "generator": {"type":  "object"},
        "impact": {"type":  "object"},
        "problemtype": {"type":  "object"},
        "references": {"type":  "object"},
        "solution": {"type":  "object"},
        "source": {"type":  "object"},
        "timeline": {"type":  "array"},
        "work_around": {"type":  "array"},
        "workaround": {"type":  "object"},
        "x_advisoryEoL": {"type":  "boolean"},
        "x_affectedList": {"type":  "array"},
        "x_likelyAffectedList": {"type":  "array"}
    },
    "required": [
        "CVE_data_meta",
        "data_format",
        "data_type",
        "data_version",
        "description"
    ],
    "type": "object"
}

Complete cve.org object schema: ./data/schemas/schema_cve_org.json

gitlab.com:

Examples:

gitlab.com schema:

{
    "properties": {
        "advisories": {
            "items": {
                "properties": {
                    "_git_import_path": {"type":  "string"},
                    "affected_range": {"type":  "string"},
                    "affected_versions": {"type":  "string"},
                    "credit": {"type":  "string"},
                    "cvss_v2": {"type":  "string"},
                    "cvss_v3": {"type":  "string"},
                    "cwe_ids": {"type":  "array"},
                    "date": {"type":  "string"},
                    "description": {"type":  "string"},
                    "fixed_versions": {"type":  "array"},
                    "identifier": {"type":  "string"},
                    "identifiers": {"type":  "array"},
                    "not_impacted": {"type":  "string"},
                    "package_slug": {"type":  "string"},
                    "pubdate": {"type":  "string"},
                    "solution": {"type":  "string"},
                    "title": {"type":  "string"},
                    "urls": {"type":  "array"},
                    "uuid": {"type":  "string"},
                    "versions": {"type":  "array"}
                },
                "required": [
                    "affected_range",
                    "affected_versions",
                    "cwe_ids",
                    "date",
                    "description",
                    "fixed_versions",
                    "identifier",
                    "identifiers",
                    "package_slug",
                    "pubdate",
                    "title",
                    "urls",
                    "uuid"
                ],
                "type": "object"
            },
            "type": "array"
        }
    },
    "required": [
        "advisories"
    ],
    "type": "object"
}

Complete gitlab.com object schema: ./data/schemas/schema_gitlab.json

nvd.nist.gov:

Examples:

nvd.nist.gov schema:

{
    "properties": {
        "configurations": {"type":  "object"},
        "cve": {"type":  "object"},
        "impact": {"type":  "object"},
        "lastModifiedDate": {"type":  "string"},
        "publishedDate": {"type":  "string"}
    },
    "required": [
        "configurations",
        "cve",
        "impact",
        "lastModifiedDate",
        "publishedDate"
    ],
    "type": "object"
}

Complete nvd.nist.gov object schema: ./data/schemas/schema_nvd.json

github.com/kurtseifried:582211:

overlay: