From ba6bc2bfa9eed50493bdcb36da2df2bfe0bb22f9 Mon Sep 17 00:00:00 2001 From: Ricky McMillen Date: Sun, 21 Mar 2021 10:41:13 +1300 Subject: [PATCH 1/8] Bump MarkupSafe to 1.1.1 in requirements.txt to fix setuptools issue As of 21 March 2021 running `make build` on the default branch of this repository breaks with the following error: ``` ERROR: Command errored out with exit status 1: command: /usr/local/bin/python -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'/tmp/pip-install-bfkg7hzw/markupsafe_2ff6d76a552d4703960c9f49f15673c8/setup.py'"'"'; __file__='"'"'/tmp/pip-install-bfkg7hzw/markupsafe_2ff6d76a552d4703960c9f49f15673c8/setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' egg_info --egg-base /tmp/pip-pip-egg-info-hn261ajv cwd: /tmp/pip-install-bfkg7hzw/markupsafe_2ff6d76a552d4703960c9f49f15673c8/ Complete output (5 lines): Traceback (most recent call last): File "", line 1, in File "/tmp/pip-install-bfkg7hzw/markupsafe_2ff6d76a552d4703960c9f49f15673c8/setup.py", line 6, in from setuptools import setup, Extension, Feature ImportError: cannot import name 'Feature' ---------------------------------------- WARNING: Discarding https://files.pythonhosted.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz#sha256=a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 (from https://pypi.org/simple/markupsafe/). Command errored out with exit status 1: python setup.py egg_info Check the logs for full command output. ERROR: Could not find a version that satisfies the requirement MarkupSafe==1.0 ERROR: No matching distribution found for MarkupSafe==1.0 ``` There is a Closed issue logged in the MarkupSafe repository, pallets/markupsafe#116 where a maintainer explained upgrading shouldn't have any compatibility issues. So the MarkupSafe requirement has been set to 1.1.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4fcc6a0..f808bfe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ Cerberus==1.1 click==6.7 Jinja2==2.10.1 -MarkupSafe==1.0 +MarkupSafe==1.1.1 psycopg2==2.7.3 PyYAML==5.2 From 46298cc3d2b96eeb6b6f57e890da200e11117f56 Mon Sep 17 00:00:00 2001 From: Ricky McMillen Date: Sun, 21 Mar 2021 12:35:55 +1300 Subject: [PATCH 2/8] wip: add roles introduced in postgres 11 --- tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4da78df..13a36b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,7 +67,9 @@ def drop_users_and_objects(cursor): WHERE rolname NOT IN ( 'test_user', 'postgres', 'pg_signal_backend', -- Roles introduced in Postgres 10: - 'pg_monitor', 'pg_read_all_settings', 'pg_read_all_stats', 'pg_stat_scan_tables' + 'pg_monitor', 'pg_read_all_settings', 'pg_read_all_stats', 'pg_stat_scan_tables', + -- Roles introduced in Postgres 11: + 'pg_execute_server_program', 'pg_read_server_files', 'pg_write_server_files' ); """) users = [u[0] for u in cursor.fetchall()] From c614f10110ab1d701ff8fc9dc3dea903bf04f4a9 Mon Sep 17 00:00:00 2001 From: Ricky McMillen Date: Sun, 21 Mar 2021 12:59:02 +1300 Subject: [PATCH 3/8] wip: update CI and Makefile with new 11.11 as supported version --- .travis.yml | 4 ++++ Makefile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bf630a5..8f5511e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,10 @@ jobs: env: SUPPORTED_PG_VERSIONS=10.4 script: sudo make test + - stage: # Intentionally left blank to parallelize + env: SUPPORTED_PG_VERSIONS=11.11 + script: sudo make test + - stage: # Intentionally left blank to parallelize install: pip install -e . -r requirements-docs.txt script: make docs diff --git a/Makefile b/Makefile index 3c8f97e..1921906 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: attach build build_tester clean coverage create_network docs psql release_pypi release_pypitest release_quay remove_network start_postgres stop_postgres test test_one_pg_version test27 test36 view_docs wait_for_postgres -SUPPORTED_PG_VERSIONS ?= 9.5.13 9.6.4 10.4 +SUPPORTED_PG_VERSIONS ?= 9.5.13 9.6.4 10.4 11.11 # The default Postgres that will be used in individual targets POSTGRES_VERSION ?= 10.4 From 819b55be0c1345416f061bfd546adcdeef103d6c Mon Sep 17 00:00:00 2001 From: Ricky McMillen Date: Sun, 21 Mar 2021 13:00:31 +1300 Subject: [PATCH 4/8] wip: update expected owner of system tables and public schema --- tests/conftest.py | 109 +++++++++++++++++++++++++++++------------- tests/test_context.py | 11 +++-- 2 files changed, 83 insertions(+), 37 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 13a36b9..fe8f86d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,43 +81,45 @@ def drop_users_and_objects(cursor): @pytest.fixture def base_spec(cursor): """ A spec with the existing state of the test database before anything has been done """ - spec = dedent(""" - postgres: - attributes: - - BYPASSRLS - - CREATEDB - - CREATEROLE - - REPLICATION - can_login: true - is_superuser: true - owns: - schemas: - - information_schema - - pg_catalog - - public - tables: - - information_schema.* - - pg_catalog.* - privileges: - schemas: - write: + cursor.execute("SELECT substring(version from 'PostgreSQL ([0-9.]*) ') FROM version()") + pg_version = int(cursor.fetchone()[0].split('.')[0]) + + # Before version 10 Postgres creates a `postgres` user plus the `test_user` + if pg_version <= 9: + spec = dedent(""" + postgres: + attributes: + - BYPASSRLS + - CREATEDB + - CREATEROLE + - REPLICATION + can_login: true + is_superuser: true + owns: + schemas: - information_schema - pg_catalog - public + tables: + - information_schema.* + - pg_catalog.* + privileges: + schemas: + write: + - information_schema + - pg_catalog + - public + + test_user: + attributes: + - PASSWORD "test_password" + can_login: yes + is_superuser: yes + """) - test_user: - attributes: - - PASSWORD "test_password" - can_login: yes - is_superuser: yes - """) - - # Postgres 10 introduces several new roles that we have to account for - cursor.execute("SELECT substring(version from 'PostgreSQL ([0-9.]*) ') FROM version()") - pg_version = cursor.fetchone()[0] - if pg_version.startswith('10.'): - spec += dedent(""" - + # In version 10 and onwards Posgres only creates the `test_user` we specify + if pg_version >= 10: + spec = dedent(""" pg_read_all_settings: pg_stat_scan_tables: @@ -129,7 +131,46 @@ def base_spec(cursor): - pg_read_all_settings - pg_stat_scan_tables - pg_read_all_stats - """) + + postgres: + attributes: + - BYPASSRLS + - CREATEDB + - CREATEROLE + - REPLICATION + can_login: true + is_superuser: true + owns: + schemas: + - information_schema + - pg_catalog + - public + tables: + - information_schema.* + - pg_catalog.* + privileges: + schemas: + write: + - information_schema + - pg_catalog + - public + + test_user: + attributes: + - PASSWORD "test_password" + can_login: yes + is_superuser: yes + """) + + # Postgres 11 introduces several new roles that we have to account for + if pg_version >= 11: + spec += dedent(""" + pg_execute_server_program: + + pg_read_server_files: + + pg_write_server_files: + """) return spec diff --git a/tests/test_context.py b/tests/test_context.py index 73247a8..5af5d38 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -14,6 +14,8 @@ TABLES = tuple('table{}'.format(i) for i in range(6)) SEQUENCES = tuple('seq{}'.format(i) for i in range(6)) DUMMY = 'foo' +TEST_USER='test_user' +POSTGRES_USER='postgres' @run_setup_sql( @@ -385,15 +387,18 @@ def test_is_superuser(all_role_attributes, expected): ]) def test_get_all_schemas_and_owners(cursor): dbcontext = context.DatabaseContext(cursor, verbose=True) + pg_version = int(dbcontext.get_version_info().postgres_version.split('.')[0]) + + expected_owner = TEST_USER if pg_version >= 11 else POSTGRES_USER expected = { common.ObjectName(SCHEMAS[0]): ROLES[0], common.ObjectName(SCHEMAS[1]): ROLES[0], common.ObjectName(SCHEMAS[2]): ROLES[1], common.ObjectName(ROLES[1]): ROLES[1], # These already existed - common.ObjectName('public'): 'postgres', - common.ObjectName('information_schema'): 'postgres', - common.ObjectName('pg_catalog'): 'postgres', + common.ObjectName('public'): expected_owner, + common.ObjectName('information_schema'): expected_owner, + common.ObjectName('pg_catalog'): expected_owner, } actual = dbcontext.get_all_schemas_and_owners() From 6530b42839596dd14d887209c9ebb83fa88d5f78 Mon Sep 17 00:00:00 2001 From: Ricky McMillen Date: Sun, 21 Mar 2021 13:38:10 +1300 Subject: [PATCH 5/8] wip: correct base spec for all versions and role memberships --- tests/conftest.py | 53 +++++++++++++++++++++--------------- tests/test_core_configure.py | 2 +- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fe8f86d..2b1b995 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,10 +83,11 @@ def base_spec(cursor): """ A spec with the existing state of the test database before anything has been done """ cursor.execute("SELECT substring(version from 'PostgreSQL ([0-9.]*) ') FROM version()") pg_version = int(cursor.fetchone()[0].split('.')[0]) + spec = "" # Before version 10 Postgres creates a `postgres` user plus the `test_user` - if pg_version <= 9: - spec = dedent(""" + if pg_version <= 10: + spec += dedent(""" postgres: attributes: - BYPASSRLS @@ -117,9 +118,28 @@ def base_spec(cursor): is_superuser: yes """) - # In version 10 and onwards Posgres only creates the `test_user` we specify - if pg_version >= 10: + # In version 10 we have some more roles to deal with + if pg_version == 10: + spec += dedent(""" + pg_monitor: + member_of: + - pg_read_all_settings + - pg_read_all_stats + - pg_stat_scan_tables + + pg_read_all_settings: + + pg_read_all_stats: + + pg_stat_scan_tables: + """) + + # In version 11 and onwards Posgres only creates the `test_user` we specify + # Postgres 11 introduces several new roles that we have to account for + if pg_version >= 11: spec = dedent(""" + pg_execute_server_program: + pg_read_all_settings: pg_stat_scan_tables: @@ -132,11 +152,16 @@ def base_spec(cursor): - pg_stat_scan_tables - pg_read_all_stats - postgres: + pg_read_server_files: + + pg_write_server_files: + + test_user: attributes: - BYPASSRLS - CREATEDB - CREATEROLE + - PASSWORD "test_password" - REPLICATION can_login: true is_superuser: true @@ -154,22 +179,6 @@ def base_spec(cursor): - information_schema - pg_catalog - public - - test_user: - attributes: - - PASSWORD "test_password" - can_login: yes - is_superuser: yes - """) - - # Postgres 11 introduces several new roles that we have to account for - if pg_version >= 11: - spec += dedent(""" - pg_execute_server_program: - - pg_read_server_files: - - pg_write_server_files: """) return spec @@ -184,7 +193,7 @@ def spec_with_new_user(tmpdir, base_spec): {new_user}: has_personal_schema: yes member_of: - - postgres + - test_user privileges: tables: read: diff --git a/tests/test_core_configure.py b/tests/test_core_configure.py index c3374dc..3a000c9 100644 --- a/tests/test_core_configure.py +++ b/tests/test_core_configure.py @@ -78,7 +78,7 @@ def test_configure_live_mode_works(capsys, cursor, spec_with_new_user, db_config assert cursor.rowcount == expected if live_mode: - cursor.execute(Q_HAS_ROLE.format(NEW_USER, 'postgres')) + cursor.execute(Q_HAS_ROLE.format(NEW_USER, 'test_user')) assert cursor.rowcount == expected cursor.execute(Q_HAS_PRIVILEGE.format(NEW_USER, 'pg_catalog.pg_class')) From 881e28d7ee6908c8dc735a8538e0ddcb969ab23c Mon Sep 17 00:00:00 2001 From: Ricky McMillen Date: Sun, 21 Mar 2021 13:48:20 +1300 Subject: [PATCH 6/8] wip: all extra role for all postgres versions 10 and above --- tests/test_context.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_context.py b/tests/test_context.py index 5af5d38..5826492 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -425,9 +425,10 @@ def test_get_all_memberships(cursor): dbcontext = context.DatabaseContext(cursor, verbose=True) expected = set([('role1', 'role0'), ('role2', 'role1')]) - pg_version = dbcontext.get_version_info().postgres_version + pg_version = int(dbcontext.get_version_info().postgres_version.split('.')[0]) + # Postgres 10 introduces several new roles and memberships that we have to account for - if pg_version.startswith('10.'): + if pg_version >= 10: expected.update(set([ ('pg_monitor', 'pg_stat_scan_tables'), ('pg_monitor', 'pg_read_all_stats'), From 5509ebd92e91cdc1a52fa9cc9c77f6d6a37f1790 Mon Sep 17 00:00:00 2001 From: Ricky McMillen Date: Sun, 21 Mar 2021 13:57:14 +1300 Subject: [PATCH 7/8] wip: don't expect postgres role to exist after version 11 add new roles --- tests/test_context.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_context.py b/tests/test_context.py index 5826492..d267b02 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -324,14 +324,27 @@ def test_get_all_personal_schemas(cursor): def test_get_all_role_attributes(cursor): dbcontext = context.DatabaseContext(cursor, verbose=True) - expected = set(['test_user', 'postgres', ROLES[0], ROLES[1]]) - pg_version = dbcontext.get_version_info().postgres_version + expected = set(['test_user', ROLES[0], ROLES[1]]) + + pg_version = int(dbcontext.get_version_info().postgres_version.split('.')[0]) + + if pg_version <= 10: + expected.update(set([ + 'postgres'] + )) + # Postgres 10 introduces several new roles that we have to account for - if pg_version.startswith('10.'): + if pg_version >= 10: expected.update(set([ 'pg_read_all_settings', 'pg_stat_scan_tables', 'pg_read_all_stats', 'pg_monitor'] )) + # Postgres 11 introduces several new roles that we have to account for + if pg_version >= 11: + expected.update(set([ + 'pg_read_server_files', 'pg_write_server_files', 'pg_execute_server_program'] + )) + actual = dbcontext.get_all_role_attributes() assert set(actual.keys()) == expected From f5662eea0aed2903e5ccf2d824f9f0d2fa7974a8 Mon Sep 17 00:00:00 2001 From: Ricky McMillen Date: Sun, 21 Mar 2021 14:05:22 +1300 Subject: [PATCH 8/8] Add PostgreSQL version 12.6 & 13.2 to CI and Makefile --- .travis.yml | 8 ++++++++ Makefile | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8f5511e..99731e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,14 @@ jobs: env: SUPPORTED_PG_VERSIONS=11.11 script: sudo make test + - stage: # Intentionally left blank to parallelize + env: SUPPORTED_PG_VERSIONS=12.6 + script: sudo make test + + - stage: # Intentionally left blank to parallelize + env: SUPPORTED_PG_VERSIONS=13.2 + script: sudo make test + - stage: # Intentionally left blank to parallelize install: pip install -e . -r requirements-docs.txt script: make docs diff --git a/Makefile b/Makefile index 1921906..5189229 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: attach build build_tester clean coverage create_network docs psql release_pypi release_pypitest release_quay remove_network start_postgres stop_postgres test test_one_pg_version test27 test36 view_docs wait_for_postgres -SUPPORTED_PG_VERSIONS ?= 9.5.13 9.6.4 10.4 11.11 +SUPPORTED_PG_VERSIONS ?= 9.5.13 9.6.4 10.4 11.11 12.6 13.2 # The default Postgres that will be used in individual targets POSTGRES_VERSION ?= 10.4