test suite and workflows

.github/workflows/build.yml

@@ -0,0 +1,26 @@
+# BUILDS AND RUNS DOCKER IMAGE
+name: Build
+
+on:
+  push:
+    branches:
+      # any branch
+      - '**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Build Docker Image
+      run: docker build -t zeek_website . --no-cache
+
+    - name: Start Docker Service
+      run: docker-compose up -d
+
+    - name: Stop Docker Service
+      run: docker-compose down
+

.github/workflows/pytest.yml

@@ -0,0 +1,31 @@
+# RUNS TEST SUITE
+name: Test Suite
+
+on:
+  push:
+    branches:
+      # any branch
+      - '**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Run Unit Tests
+      run: |
+        cd zeek-package-website && pytest -s --showlocals tests/test_parser.py tests/test_search.py tests/test_scraper.py
+

README.md

@@ -1,3 +1,7 @@
+[![Build](https://github.com/zeek/package-website/actions/workflows/build.yml/badge.svg)](https://github.com/zeek/package-website/actions/workflows/build.yml)
+[![Test Suite](https://github.com/zeek/package-website/actions/workflows/pytest.yml/badge.svg)](https://github.com/zeek/package-website/actions/workflows/pytest.yml)
+[![license](https://img.shields.io/github/license/zeek/package-website?color=23228B22)](https://github.com/zeek/package-website/blob/main/LICENSE)
+
 # Zeek Package Website Repository
 ...
 

requirements.txt

@@ -12,3 +12,4 @@ fastapi_utils
 mistune
 markdown
 pytest
+requests

zeek-package-website/app/api/parser.py

@@ -60,33 +60,34 @@ def parse_data(self) -> dict:
 
             # extract our desired fields
             # TODO: pass in header + text to look for
-            self.author = self.get_line("credits", header)
-            self.description = self.get_line("description", header)
-            self.tags = self.get_line("tags", header)
-            self.version = self.get_line("version", header)
-            self.get_next("depends", header)
-            self.test_cmd = self.get_line("test_command", header)
-            self.build_cmd = self.get_line("build_command", header)
-            self.url = self.get_line("url", header)
-            self.summary = self.get_line("summary", header)
-            self.script_dir = self.get_line("script_dir", header)
-            self.plugin_dir = self.get_line("plugin_dir", header)
-            self.readme = self.get_readme()
+            self.author         = self.get_line("credits", header)
+            self.description    = self.get_line("description", header)
+            self.tags           = self.get_line("tags", header)
+            self.version        = self.get_line("version", header)
+            self.depends        = self.get_next("depends", header)
+            self.test_cmd       = self.get_line("test_command", header)
+            self.build_cmd      = self.get_line("build_command", header)
+            self.url            = self.get_line("url", header)
+            self.summary        = self.get_line("summary", header)
+            self.script_dir     = self.get_line("script_dir", header)
+            self.plugin_dir     = self.get_line("plugin_dir", header)
+            self.readme         = self.get_readme()
+
             if self.readme is not None and self.url is not None:
                 self.get_images()
 
             self.pkg_dict[self.section_header] = {
-                "description": self.description,
-                "tags": self.tags,
-                "version": self.version,
-                "depends": self.depends,
-                "test_cmd": self.test_cmd,
-                "build_cmd": self.build_cmd,
-                "url": self.url,
-                "summary": self.summary,
-                "script_dir": self.script_dir,
-                "plugin_dir": self.plugin_dir,
-                "readme": self.readme
+                "description"   : self.description,
+                "tags"          : self.tags,
+                "version"       : self.version,
+                "depends"       : self.depends,
+                "test_cmd"      : self.test_cmd,
+                "build_cmd"     : self.build_cmd,
+                "url"           : self.url,
+                "summary"       : self.summary,
+                "script_dir"    : self.script_dir,
+                "plugin_dir"    : self.plugin_dir,
+                "readme"        : self.readme
             }
 
 
@@ -95,8 +96,6 @@ def parse_data(self) -> dict:
 
         return self.pkg_dict
 
-
-
     def get_name(self) -> str:
         """
         @brief Finds all section headers in the file contents.
@@ -148,8 +147,8 @@ def get_next(self, text, header) -> list:
 
         if next_match:
             next_line = next_match.group(1).strip().split('\n')
-            # remove tabs
-            next_line = [ln.replace('\t', '') for ln in next_line]
+            # remove tabs and leading spaces
+            next_line = [ln.replace('\t', '').lstrip() for ln in next_line]
         else:
             next_line = None
 

zeek-package-website/app/api/readme_scraper.py

@@ -33,6 +33,9 @@ def load_packages() -> []:
 def find_missing(package: dict) -> []:
     missing = []
 
+    if(package is None):
+        return None
+
     for key in package.keys():
         if package[key] is None and key in fields.keys():
             missing.append(key)
@@ -42,6 +45,9 @@ def find_missing(package: dict) -> []:
 
 def get_field(readme: str, field: str) -> str:
 
+    if readme is None or field is None:
+        return None
+
     search_term = f"{fields[field]}"
 
     field = re.search(f'(?<=# {search_term})(.*)(?=\n)', readme.lower())

zeek-package-website/app/api/search/json_files/CVE-2021-42292.json

@@ -1 +1 @@
-{"description": "A package to detect CVE-2021-42292, a Microsoft Excel priviledge exploit.", "tags": null, "version": "v0.1.0", "depends": null, "test_cmd": "cd testing && btest -c btest.cfg", "build_cmd": null, "url": "https://github.com/corelight/CVE-2021-42292", "summary": "A package to detect CVE-2021-42292, a Microsoft Excel priviledge exploit.", "script_dir": "scripts", "plugin_dir": null, "readme": "## CVE-2021-42292\n\nThis package will detect exploits of [CVE-2021-42292](https://raw.githubusercontent.com/corelight/CVE-2021-42292/master/http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-42292), a Microsoft Excel local\nprivilege escalation vulnerability, and generate a notice in notice.log for it.\n\nhttps://corelight.com/blog/detecting-cve-2021-42292  \n\n#### Detection Method:\n\nThis package detects the vulnerability when the triggering Excel spreadsheet downloads a second spreadsheet.\nThe second spreadsheet is executed with elevated privileges.  We can detect Microsoft Excel downloading\na Microsoft Excel file with this script.  In our testing on some live networks we monitor,\nthis combination was extremely rare and we have not seen any false positives so far.\n\n#### Usage:\n\n```\n$ zeek -Cr excelsploit_1.pcap packages\n\n$ cat notice.log\n#separator \\x09\n#set_separator  ,\n#empty_field    (empty)\n#unset_field    -\n#path   notice\n#open   2021-11-10-10-56-50\n#fields ts      uid     id.orig_h       id.orig_p       id.resp_h       id.resp_p       fuid    file_mime_type  file_desc       proto   note    msg     sub     src     dst     p       n       peer_descr      actions email_dest      suppress_for    remote_location.country_code    remote_location.region  remote_location.city    remote_location.latitude        remote_location.longitude\n#types  time    string  addr    port    addr    port    string  string  string  enum    enum    string  string  addr    addr    port    count   string  set[enum]       set[string]     interval        string  string  string  double  double\n1636433584.277654       CeV1DA2EM1pRTfgWkc      127.0.0.1       51543   127.0.0.1       80      -       -       -       tcp     CVE_2021_42292::CVE_2021_42292  127.0.0.1 may be compromised by CVE-2021-42292, MS Office Excel download using Office from 127.0.0.1 detected. See sub field for additional triage information  host='127.0.0.1', method='HEAD', user_agent='Microsoft Office Excel 2014', CONTENT-TYPE='application/vnd.ms-excel', uri='/replica.xls'      127.0.0.1       127.0.0.1       80      -       -       Notice::ACTION_LOG      (empty) 3600.000000     -       -       -       -       -\n1636433584.311236       CgKWSM1bhhl7K8B6n8      127.0.0.1       51545   127.0.0.1       80      -       -       -       tcp     CVE_2021_42292::CVE_2021_42292  127.0.0.1 may be compromised by CVE-2021-42292, MS Office Excel download using Office from 127.0.0.1 detected. See sub field for additional triage information  host='127.0.0.1', method='GET', user_agent='Mozilla/4.0 (compatible; ms-office; MSOffice 16)', CONTENT-TYPE='application/vnd.ms-excel', uri='/replica.xls'  127.0.0.1       127.0.0.1       80      -       -       Notice::ACTION_LOG      (empty) 3600.000000     -       -       -       -       -\n#close  2021-11-10-10-56-50\n```\n\nSuricata rules are also provided that mirror the detection methodology of the\nZeek package.\n\n#### Links:\n* Associated blog including walk through of code elements:  \n    * https://corelight.com/blog/detecting-cve-2021-42292   \n* MIME Types:\n    * https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types\n* Excel User Agents:\n    * https://developers.whatismybrowser.com/useragents/explore/software_name/excel/\n"}
+{"description": "A package to detect CVE-2021-42292, a Microsoft Excel priviledge exploit.", "tags": null, "version": "v0.1.0", "depends": ["zeek >=3.0.0"], "test_cmd": "cd testing && btest -c btest.cfg", "build_cmd": null, "url": "https://github.com/corelight/CVE-2021-42292", "summary": "A package to detect CVE-2021-42292, a Microsoft Excel priviledge exploit.", "script_dir": "scripts", "plugin_dir": null, "readme": "## CVE-2021-42292\n\nThis package will detect exploits of [CVE-2021-42292](https://raw.githubusercontent.com/corelight/CVE-2021-42292/master/http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-42292), a Microsoft Excel local\nprivilege escalation vulnerability, and generate a notice in notice.log for it.\n\nhttps://corelight.com/blog/detecting-cve-2021-42292  \n\n#### Detection Method:\n\nThis package detects the vulnerability when the triggering Excel spreadsheet downloads a second spreadsheet.\nThe second spreadsheet is executed with elevated privileges.  We can detect Microsoft Excel downloading\na Microsoft Excel file with this script.  In our testing on some live networks we monitor,\nthis combination was extremely rare and we have not seen any false positives so far.\n\n#### Usage:\n\n```\n$ zeek -Cr excelsploit_1.pcap packages\n\n$ cat notice.log\n#separator \\x09\n#set_separator  ,\n#empty_field    (empty)\n#unset_field    -\n#path   notice\n#open   2021-11-10-10-56-50\n#fields ts      uid     id.orig_h       id.orig_p       id.resp_h       id.resp_p       fuid    file_mime_type  file_desc       proto   note    msg     sub     src     dst     p       n       peer_descr      actions email_dest      suppress_for    remote_location.country_code    remote_location.region  remote_location.city    remote_location.latitude        remote_location.longitude\n#types  time    string  addr    port    addr    port    string  string  string  enum    enum    string  string  addr    addr    port    count   string  set[enum]       set[string]     interval        string  string  string  double  double\n1636433584.277654       CeV1DA2EM1pRTfgWkc      127.0.0.1       51543   127.0.0.1       80      -       -       -       tcp     CVE_2021_42292::CVE_2021_42292  127.0.0.1 may be compromised by CVE-2021-42292, MS Office Excel download using Office from 127.0.0.1 detected. See sub field for additional triage information  host='127.0.0.1', method='HEAD', user_agent='Microsoft Office Excel 2014', CONTENT-TYPE='application/vnd.ms-excel', uri='/replica.xls'      127.0.0.1       127.0.0.1       80      -       -       Notice::ACTION_LOG      (empty) 3600.000000     -       -       -       -       -\n1636433584.311236       CgKWSM1bhhl7K8B6n8      127.0.0.1       51545   127.0.0.1       80      -       -       -       tcp     CVE_2021_42292::CVE_2021_42292  127.0.0.1 may be compromised by CVE-2021-42292, MS Office Excel download using Office from 127.0.0.1 detected. See sub field for additional triage information  host='127.0.0.1', method='GET', user_agent='Mozilla/4.0 (compatible; ms-office; MSOffice 16)', CONTENT-TYPE='application/vnd.ms-excel', uri='/replica.xls'  127.0.0.1       127.0.0.1       80      -       -       Notice::ACTION_LOG      (empty) 3600.000000     -       -       -       -       -\n#close  2021-11-10-10-56-50\n```\n\nSuricata rules are also provided that mirror the detection methodology of the\nZeek package.\n\n#### Links:\n* Associated blog including walk through of code elements:  \n    * https://corelight.com/blog/detecting-cve-2021-42292   \n* MIME Types:\n    * https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types\n* Excel User Agents:\n    * https://developers.whatismybrowser.com/useragents/explore/software_name/excel/\n"}

zeek-package-website/app/api/search/json_files/CVE-2022-23270-PPTP.json

@@ -1 +1 @@
-{"description": "A package to detect CVE-2022-23270.", "tags": null, "version": "master", "depends": null, "test_cmd": "cd testing && btest -c btest.cfg", "build_cmd": null, "url": "https://github.com/corelight/CVE-2022-23270-PPTP", "summary": "A package to detect CVE-2022-23270.", "script_dir": "scripts", "plugin_dir": null, "readme": "# CVE-2022-23270\n\nA package to detect CVE-2022-23270, a vulnerability in Microsoft's PPTP implementation.\n\n## Example\n\nYou can run this logic on the included PCAP in the `testing\\traces` directory:\n\n```\n$ zeek -Cr CVE-2022-23270-exploited.pcap packages\n\n$ cat notice.log \n#separator \\x09\n#set_separator\t,\n#empty_field\t(empty)\n#unset_field\t-\n#path\tnotice\n#open\t2022-05-10-23-03-47\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tfuid\tfile_mime_type\tfile_desc\tproto\tnote\tmsg\tsub\tsrc\tdst\tp\tn\tpeer_descr\tactions\temail_dest\tsuppress_for\tremote_location.country_code\tremote_location.region\tremote_location.city\tremote_location.latitude\tremote_location.longitude\n#types\ttime\tstring\taddr\tport\taddr\tport\tstring\tstring\tstring\tenum\tenum\tstring\tstring\taddr\taddr\tport\tcount\tstring\tset[enum]\tset[string]\tinterval\tstring\tstring\tstring\tdouble\tdouble\n1652212222.744235\tCHhAvVGS1DHFjwGM9\t192.168.88.166\t51143\t192.168.88.157\t1723\t-\t-\t-\ttcp\tCVE202223270::CVE_2022_23270_Attempt\tPotential PPTP CVE-2022-23270 exploit attempt: 192.168.88.166 attempted exploit against 192.168.88.157\t-\t192.168.88.166\t192.168.88.157\t1723\t-\t-\tNotice::ACTION_LOG\t(empty)\t3600.000000\t-\t-\t-\t-\t-\n1652212222.744235\tCHhAvVGS1DHFjwGM9\t192.168.88.166\t51143\t192.168.88.157\t1723\t-\t-\t-\ttcp\tCVE202223270::CVE_2022_23270_Success\tPPTP CVE-2022-23270 exploit success: 192.168.88.166 exploited 192.168.88.157\t-\t192.168.88.166\t192.168.88.157\t1723\t-\t-\tNotice::ACTION_LOG\t(empty)\t3600.000000\t-\t-\t-\t-\t-\n#close\t2022-05-10-23-03-47\n```\n\n## RFCs\n- https://datatracker.ietf.org/doc/html/rfc2637"}
+{"description": "A package to detect CVE-2022-23270.", "tags": null, "version": "master", "depends": ["zeek >=4.0.0"], "test_cmd": "cd testing && btest -c btest.cfg", "build_cmd": null, "url": "https://github.com/corelight/CVE-2022-23270-PPTP", "summary": "A package to detect CVE-2022-23270.", "script_dir": "scripts", "plugin_dir": null, "readme": "# CVE-2022-23270\n\nA package to detect CVE-2022-23270, a vulnerability in Microsoft's PPTP implementation.\n\n## Example\n\nYou can run this logic on the included PCAP in the `testing\\traces` directory:\n\n```\n$ zeek -Cr CVE-2022-23270-exploited.pcap packages\n\n$ cat notice.log \n#separator \\x09\n#set_separator\t,\n#empty_field\t(empty)\n#unset_field\t-\n#path\tnotice\n#open\t2022-05-10-23-03-47\n#fields\tts\tuid\tid.orig_h\tid.orig_p\tid.resp_h\tid.resp_p\tfuid\tfile_mime_type\tfile_desc\tproto\tnote\tmsg\tsub\tsrc\tdst\tp\tn\tpeer_descr\tactions\temail_dest\tsuppress_for\tremote_location.country_code\tremote_location.region\tremote_location.city\tremote_location.latitude\tremote_location.longitude\n#types\ttime\tstring\taddr\tport\taddr\tport\tstring\tstring\tstring\tenum\tenum\tstring\tstring\taddr\taddr\tport\tcount\tstring\tset[enum]\tset[string]\tinterval\tstring\tstring\tstring\tdouble\tdouble\n1652212222.744235\tCHhAvVGS1DHFjwGM9\t192.168.88.166\t51143\t192.168.88.157\t1723\t-\t-\t-\ttcp\tCVE202223270::CVE_2022_23270_Attempt\tPotential PPTP CVE-2022-23270 exploit attempt: 192.168.88.166 attempted exploit against 192.168.88.157\t-\t192.168.88.166\t192.168.88.157\t1723\t-\t-\tNotice::ACTION_LOG\t(empty)\t3600.000000\t-\t-\t-\t-\t-\n1652212222.744235\tCHhAvVGS1DHFjwGM9\t192.168.88.166\t51143\t192.168.88.157\t1723\t-\t-\t-\ttcp\tCVE202223270::CVE_2022_23270_Success\tPPTP CVE-2022-23270 exploit success: 192.168.88.166 exploited 192.168.88.157\t-\t192.168.88.166\t192.168.88.157\t1723\t-\t-\tNotice::ACTION_LOG\t(empty)\t3600.000000\t-\t-\t-\t-\t-\n#close\t2022-05-10-23-03-47\n```\n\n## RFCs\n- https://datatracker.ietf.org/doc/html/rfc2637"}