diff --git a/httpobs/conf/__init__.py b/httpobs/conf/__init__.py index 40d03c1..9549e5a 100644 --- a/httpobs/conf/__init__.py +++ b/httpobs/conf/__init__.py @@ -74,10 +74,6 @@ def __conf(section, param, type=None, default=None): # Scanner configuration SCANNER_ABORT_SCAN_TIME = int(environ.get('HTTPOBS_SCANNER_ABORT_SCAN_TIME') or __conf('scanner', 'abort_scan_time')) -SCANNER_MOZILLA_DOMAINS = [ - domain.strip() - for domain in (environ.get('HTTPOBS_SCANNER_MOZILLA_DOMAINS') or __conf('scanner', 'mozilla_domains')).split(',') -] SCANNER_PINNED_DOMAINS = [ domain.strip() for domain in (environ.get('HTTPOBS_SCANNER_PINNED_DOMAINS') or __conf('scanner', 'pinned_domains')).split(',') diff --git a/httpobs/conf/httpobs.conf b/httpobs/conf/httpobs.conf index a4bb3d9..97f4819 100644 --- a/httpobs/conf/httpobs.conf +++ b/httpobs/conf/httpobs.conf @@ -25,5 +25,4 @@ user_agent = Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:98.0) Gecko/201001 [scanner] abort_scan_time = 1800 -mozilla_domains = mozilla,allizom,browserid,firefox,persona,taskcluster,webmaker pinned_domains = accounts.firefox.com,addons.mozilla.org,aus4.mozilla.org,aus5.mozilla.org,cdn.mozilla.org,services.mozilla.com diff --git a/httpobs/docs/api.md b/httpobs/docs/api.md index 582b505..5621bad 100644 --- a/httpobs/docs/api.md +++ b/httpobs/docs/api.md @@ -321,43 +321,6 @@ Example: "score_description": "Content Security Policy (CSP) implemented with unsafe-inline inside style-src directive", "score_modifier": -5 }, - "contribute": { - "expectation": "contribute-json-with-required-keys", - "name": "contribute", - "output": { - "data": { - "bugs": { - "list": "https://github.com/mozilla/addons-server/issues", - "report": "https://github.com/mozilla/addons-server/issues/new" - }, - "description": "Mozilla's official site for add-ons to Mozilla software, such as Firefox, Thunderbird, and SeaMonkey.", - "name": "Olympia", - "participate": { - "docs": "http://addons-server.readthedocs.org/", - "home": "https://wiki.mozilla.org/Add-ons/Contribute/AMO/Code", - "irc": "irc://irc.mozilla.org/#amo", - "irc-contacts": [ - "andym", - "cgrebs", - "kumar", - "magopian", - "mstriemer", - "muffinresearch", - "tofumatt" - ] - }, - "urls": { - "dev": "https://addons-dev.allizom.org/", - "prod": "https://addons.mozilla.org/", - "stage": "https://addons.allizom.org/" - } - } - }, - "pass": true, - "result": "contribute-json-with-required-keys", - "score_description": "Contribute.json implemented with the required contact information", - "score_modifier": 0 - }, "cookies": { "expectation": "cookies-secure-with-httponly-sessions", "name": "cookies", diff --git a/httpobs/docs/scoring.md b/httpobs/docs/scoring.md index 2c5b849..5573d2f 100644 --- a/httpobs/docs/scoring.md +++ b/httpobs/docs/scoring.md @@ -27,15 +27,6 @@ Scoring Range | Grade ## Score Modifiers -[Contribute.json](https://www.contributejson.org/) | Description | Modifier ---- | --- | :---: -contribute-json-only-required-on-mozilla-properties | Contribute.json isn't required on websites that don't belong to Mozilla | 0 -contribute-json-with-required-keys | Contribute.json implemented with the required contact information | 0 -contribute-json-missing-required-keys | Contribute.json exists, but is missing some of the required keys | -5 -contribute-json-not-implemented | Contribute.json file missing from root of website | -5 -contribute-json-invalid-json | Contribute.json file cannot be parsed | -10 -
- [Cookies](https://infosec.mozilla.org/guidelines/web_security#cookies) | Description | Modifier --- | --- | :---: cookies-secure-with-httponly-sessions-and-samesite | All cookies use the Secure flag, session cookies use the HttpOnly flag, and cross-origin restrictions are in place via the SameSite flag | 5 diff --git a/httpobs/scanner/analyzer/__init__.py b/httpobs/scanner/analyzer/__init__.py index a3e7d1a..a0af98d 100644 --- a/httpobs/scanner/analyzer/__init__.py +++ b/httpobs/scanner/analyzer/__init__.py @@ -1,4 +1,4 @@ -from .content import contribute, subresource_integrity +from .content import subresource_integrity from .headers import ( content_security_policy, cookies, @@ -15,7 +15,6 @@ tests = ( content_security_policy, cookies, - contribute, cross_origin_resource_sharing, redirection, referrer_policy, diff --git a/httpobs/scanner/analyzer/content.py b/httpobs/scanner/analyzer/content.py index 6b0e060..3a57b85 100644 --- a/httpobs/scanner/analyzer/content.py +++ b/httpobs/scanner/analyzer/content.py @@ -4,7 +4,6 @@ from bs4 import BeautifulSoup as bs from publicsuffixlist import PublicSuffixList -from httpobs.conf import SCANNER_MOZILLA_DOMAINS from httpobs.scanner.analyzer.decorators import scored_test from httpobs.scanner.analyzer.utils import only_if_worse from httpobs.scanner.retriever.retriever import HTML_TYPES @@ -14,80 +13,6 @@ json.JSONDecodeError = ValueError -@scored_test -def contribute(reqs: dict, expectation='contribute-json-with-required-keys') -> dict: - """ - :param reqs: dictionary containing all the request and response objects - :param expectation: test expectation - contribute-json-with-required-keys: contribute.json exists, with all the required_keys [default] - contribute-json-missing-required-keys: contribute.json exists, but missing some of the required_keys (A-) - contribute-json-only-required-on-mozilla-properties: contribute.json isn't required, - since it's not a Mozilla domain - contribute-json-not-implemented: contribute.json file missing (B+) - :return: dictionary with: - data: the parsed contribute.json file - expectation: test expectation - pass: whether the site's configuration met its expectation (null for non-Mozilla sites) - result: short string describing the result of the test - """ - # TODO: allow a bonus if you have a contribute.json on a non-Mozilla website - - output = { - 'data': None, - 'expectation': expectation, - 'pass': False, - 'result': None, - } - - # The keys that are required to be in contribute.json - required_keys = ('name', 'description', 'participate', 'bugs', 'urls') - - response = reqs['responses']['auto'] - - # This finds the SLD ('mozilla' out of 'mozilla.org') if it exists - if '.' in urlparse(response.url).netloc: - second_level_domain = urlparse(response.url).netloc.split('.')[-2] - else: - second_level_domain = '' - - if second_level_domain not in SCANNER_MOZILLA_DOMAINS: - output['expectation'] = output['result'] = 'contribute-json-only-required-on-mozilla-properties' - - # If there's a contribute.json file - elif reqs['resources']['/contribute.json']: - try: - contrib = json.loads(reqs['resources']['/contribute.json']) - - if all(key in contrib for key in required_keys): - output['result'] = 'contribute-json-with-required-keys' - else: - output['result'] = 'contribute-json-missing-required-keys' - except (json.JSONDecodeError, TypeError): - contrib = {} - output['result'] = 'contribute-json-invalid-json' - - # Store the contribute.json file - if any(key in contrib for key in required_keys): - contrib = {key: contrib.get(key) for key in required_keys if key in contrib} - - # Store contribute.json in the database if it's under a certain size - if len(str(contrib)) < 32768: - output['data'] = contrib - else: - output['data'] = {} - - else: - output['result'] = 'contribute-json-not-implemented' - - # Check to see if the test passed or failed - if expectation == output['result']: - output['pass'] = True - elif output['result'] == 'contribute-json-only-required-on-mozilla-properties': - output['pass'] = True - - return output - - @scored_test def subresource_integrity(reqs: dict, expectation='sri-implemented-and-external-scripts-loaded-securely') -> dict: """ diff --git a/httpobs/scanner/grader/grade.py b/httpobs/scanner/grader/grade.py index e6133ef..0d664e5 100644 --- a/httpobs/scanner/grader/grade.py +++ b/httpobs/scanner/grader/grade.py @@ -34,27 +34,6 @@ GRADES = set(GRADE_CHART.values()) SCORE_TABLE = { - # contribute.json - 'contribute-json-with-required-keys': { - 'description': 'Contribute.json implemented with the required contact information', - 'modifier': 0, - }, - 'contribute-json-only-required-on-mozilla-properties': { - 'description': 'Contribute.json isn\'t required on websites that don\'t belong to Mozilla', - 'modifier': 0, - }, - 'contribute-json-missing-required-keys': { - 'description': 'Contribute.json exists, but is missing some of the required keys', - 'modifier': -5, - }, - 'contribute-json-not-implemented': { - 'description': 'Contribute.json file missing from root of website', - 'modifier': -5, - }, - 'contribute-json-invalid-json': { - 'description': 'Contribute.json file cannot be parsed', - 'modifier': -10, - }, # CSP 'csp-implemented-with-no-unsafe-default-src-none': { 'description': 'Content Security Policy (CSP) implemented with default-src \'none\' and no \'unsafe\'', diff --git a/httpobs/scanner/retriever/retriever.py b/httpobs/scanner/retriever/retriever.py index c8e9b13..523fe28 100644 --- a/httpobs/scanner/retriever/retriever.py +++ b/httpobs/scanner/retriever/retriever.py @@ -142,7 +142,7 @@ def retrieve_all(hostname, **kwargs): } # The list of resources to get - resources = ('/clientaccesspolicy.xml', '/contribute.json', '/crossdomain.xml', '/robots.txt') + resources = ('/clientaccesspolicy.xml', '/crossdomain.xml', '/robots.txt') # Create some reusable sessions, one for HTTP and one for HTTPS http_session = __create_session('http://' + hostname + kwargs['http_port'] + kwargs['path'], **kwargs) diff --git a/httpobs/tests/unittests/test_content.py b/httpobs/tests/unittests/test_content.py index 87377cd..d9ee31b 100644 --- a/httpobs/tests/unittests/test_content.py +++ b/httpobs/tests/unittests/test_content.py @@ -1,135 +1,9 @@ from unittest import TestCase -from httpobs.scanner.analyzer.content import contribute, subresource_integrity +from httpobs.scanner.analyzer.content import subresource_integrity from httpobs.tests.utils import empty_requests -class TestContribute(TestCase): - def setUp(self): - self.reqs = empty_requests() - - def tearDown(self): - self.reqs = None - - def test_no_contribute_mozilla(self): - result = contribute(self.reqs) - - self.assertEquals('contribute-json-not-implemented', result['result']) - self.assertFalse(result['pass']) - - def test_no_contribute_not_mozilla(self): - self.reqs['responses']['auto'].url = 'https://github.com' - - result = contribute(self.reqs) - - self.assertEquals('contribute-json-only-required-on-mozilla-properties', result['result']) - self.assertTrue(result['pass']) - - def test_invalid_json(self): - self.reqs['resources']['/contribute.json'] = 'foobar' - - result = contribute(self.reqs) - - self.assertEquals('contribute-json-invalid-json', result['result']) - self.assertFalse(result['pass']) - - def test_contribute_too_large(self): - self.reqs['resources']['/contribute.json'] = '{"name": "' + 'foo' * 100000 + '"}' - - result = contribute(self.reqs) - - self.assertEquals(result['data'], {}) - - def test_with_required_keys(self): - self.reqs['resources'][ - '/contribute.json' - ] = """ - { - "name": "Bedrock", - "description": "The app powering www.mozilla.org.", - "repository": { - "url": "https://github.com/mozilla/bedrock", - "license": "MPL2", - "tests": "https://travis-ci.org/mozilla/bedrock/" - }, - "participate": { - "home": "https://wiki.mozilla.org/Webdev/GetInvolved/mozilla.org", - "docs": "http://bedrock.readthedocs.org/", - "mailing-list": "https://www.mozilla.org/about/forums/#dev-mozilla-org", - "irc": "irc://irc.mozilla.org/#www" - }, - "bugs": { - "list": "https://bugzilla.mozilla.org/describecomponents.cgi?product=www.mozilla.org", - "report": "https://bugzilla.mozilla.org/enter_bug.cgi?product=www.mozilla.org", - "mentored": "https://bugzilla.mozilla.org/buglist.cgi?f1=bug_mentor&o1=..." - }, - "urls": { - "prod": "https://www.mozilla.org", - "stage": "https://www.allizom.org", - "dev": "https://www-dev.allizom.org", - "demo1": "https://www-demo1.allizom.org", - "demo2": "https://www-demo2.allizom.org", - "demo3": "https://www-demo3.allizom.org", - "demo4": "https://www-demo4.allizom.org", - "demo5": "https://www-demo5.allizom.org" - }, - "keywords": [ - "python", - "less-css", - "django", - "html5", - "jquery" - ] - }""" - - result = contribute(self.reqs) - - self.assertEquals('contribute-json-with-required-keys', result['result']) - self.assertTrue(result['pass']) - - def test_missing_required_keys(self): - self.reqs['resources'][ - '/contribute.json' - ] = """ - { - "name": "Bedrock", - "description": "The app powering www.mozilla.org.", - "repository": { - "url": "https://github.com/mozilla/bedrock", - "license": "MPL2", - "tests": "https://travis-ci.org/mozilla/bedrock/" - }, - "participate": { - "home": "https://wiki.mozilla.org/Webdev/GetInvolved/mozilla.org", - "docs": "http://bedrock.readthedocs.org/", - "mailing-list": "https://www.mozilla.org/about/forums/#dev-mozilla-org", - "irc": "irc://irc.mozilla.org/#www" - }, - "urls": { - "prod": "https://www.mozilla.org", - "stage": "https://www.allizom.org", - "dev": "https://www-dev.allizom.org", - "demo1": "https://www-demo1.allizom.org", - "demo2": "https://www-demo2.allizom.org", - "demo3": "https://www-demo3.allizom.org", - "demo4": "https://www-demo4.allizom.org", - "demo5": "https://www-demo5.allizom.org" - }, - "keywords": [ - "python", - "less-css", - "django", - "html5", - "jquery" - ] - }""" - - result = contribute(self.reqs) - - self.assertEquals('contribute-json-missing-required-keys', result['result']) - self.assertFalse(result['pass']) - - class TestSubResourceIntegrity(TestCase): def setUp(self): self.reqs = empty_requests() diff --git a/httpobs/tests/unittests/test_grades.py b/httpobs/tests/unittests/test_grades.py index 9b02904..e06b4b6 100644 --- a/httpobs/tests/unittests/test_grades.py +++ b/httpobs/tests/unittests/test_grades.py @@ -6,9 +6,9 @@ class TestGrader(TestCase): def test_get_score_description(self): self.assertEquals( - 'Contribute.json implemented with the required contact information', - get_score_description('contribute-json-with-required-keys'), + 'Content Security Policy (CSP) header not implemented', + get_score_description('csp-not-implemented'), ) def test_get_score_modifier(self): - self.assertEquals(0, get_score_modifier('contribute-json-with-required-keys')) + self.assertEquals(-25, get_score_modifier('csp-not-implemented')) diff --git a/httpobs/tests/unittests/test_retriever.py b/httpobs/tests/unittests/test_retriever.py index 6107c8e..de917c7 100644 --- a/httpobs/tests/unittests/test_retriever.py +++ b/httpobs/tests/unittests/test_retriever.py @@ -27,7 +27,6 @@ def test_retrieve_mdn(self): # Various things we know about developer.mozilla.org self.assertIsNotNone(reqs['resources']['__path__']) - self.assertIsNotNone(reqs['resources']['/contribute.json']) self.assertIsNotNone(reqs['resources']['/robots.txt']) self.assertIsNone(reqs['resources']['/clientaccesspolicy.xml']) self.assertIsNone(reqs['resources']['/crossdomain.xml']) diff --git a/httpobs/tests/utils.py b/httpobs/tests/utils.py index 8e2dcb8..603531e 100644 --- a/httpobs/tests/utils.py +++ b/httpobs/tests/utils.py @@ -17,7 +17,6 @@ def empty_requests(http_equiv_file=None) -> dict: '__path__': None, '/': None, '/clientaccesspolicy.xml': None, - '/contribute.json': None, '/crossdomain.xml': None, '/robots.txt': None, },