From 0dabd59d1770cb79accf6552400cb5a7a4cb62bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez?= Date: Tue, 29 May 2018 14:22:28 -0500 Subject: [PATCH 1/2] Change psql concurrency from autocommit to serializable. Autocommit is giving concurrency errors in PostgreSQL when operations are sent in parallel. Using serializable transactions seems to fix it. For ex: ERROR: deadlock detected DETAIL: Process 176184 waits for ShareLock on transaction 15529683; blocked by process 191002. Process 191002 waits for ShareLock on transaction 15529684; blocked by process 178678. Process 178678 waits for ExclusiveLock on tuple (1386,16) of relation 43815 of database 16391; blocked by process 176184. Process 176184: DELETE FROM ip_net_plan AS p WHERE vrf_id = 0 AND prefix = '10.0.10.240/28' Process 191002: DELETE FROM ip_net_plan AS p WHERE vrf_id = 0 AND prefix = '10.0.11.0/28' Process 178678: DELETE FROM ip_net_plan AS p WHERE vrf_id = 0 AND prefix = '10.0.10.208/28' HINT: See server log for query details. CONTEXT: while locking tuple (1386,16) in relation "ip_net_plan" SQL statement "UPDATE ip_net_plan SET children = (SELECT COUNT(1) FROM ip_net_plan WHERE vrf_id = OLD.vrf_id AND iprange(prefix) << iprange(old_parent.prefix) AND indent = old_parent.indent+1) WHERE id = old_parent.id" PL/pgSQL function tf_ip_net_plan__prefix_iu_after() line 92 at SQL statement STATEMENT: DELETE FROM ip_net_plan AS p WHERE vrf_id = 0 AND prefix = '10.0.10.240/28' --- nipap/nipap/backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nipap/nipap/backend.py b/nipap/nipap/backend.py index f8b309226..67a1f5b36 100644 --- a/nipap/nipap/backend.py +++ b/nipap/nipap/backend.py @@ -771,7 +771,7 @@ def _connect_db(self): while True: try: self._con_pg = psycopg2.connect(**db_args) - self._con_pg.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + self._con_pg.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) self._curs_pg = self._con_pg.cursor(cursor_factory=psycopg2.extras.DictCursor) self._register_inet() psycopg2.extras.register_hstore(self._con_pg, globally=True, unicode=True) From f2554dd91d60a777500b09872555492fd4d8a2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20S=C3=A1nchez?= Date: Sat, 30 Jun 2018 21:35:46 -0500 Subject: [PATCH 2/2] Use locking in the DELETES to avoid deadlocks --- nipap/nipap/backend.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/nipap/nipap/backend.py b/nipap/nipap/backend.py index 67a1f5b36..365b0ab32 100644 --- a/nipap/nipap/backend.py +++ b/nipap/nipap/backend.py @@ -771,7 +771,7 @@ def _connect_db(self): while True: try: self._con_pg = psycopg2.connect(**db_args) - self._con_pg.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) + self._con_pg.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) self._curs_pg = self._con_pg.cursor(cursor_factory=psycopg2.extras.DictCursor) self._register_inet() psycopg2.extras.register_hstore(self._con_pg, globally=True, unicode=True) @@ -1335,7 +1335,10 @@ def remove_vrf(self, auth, spec): self.remove_prefix(auth, spec = v6spec, recursive = True) where, params = self._expand_vrf_spec(spec) - sql = "DELETE FROM ip_net_vrf WHERE %s" % where + sql = """LOCK TABLE ip_net_plan IN EXCLUSIVE MODE; +LOCK TABLE ip_net_vrf IN EXCLUSIVE MODE; +LOCK TABLE ip_net_pool IN EXCLUSIVE MODE; +DELETE FROM ip_net_vrf WHERE %s""" % where self._execute(sql, params) # write to audit table @@ -1875,7 +1878,10 @@ def remove_pool(self, auth, spec): pools = self.list_pool(auth, spec) where, params = self._expand_pool_spec(spec) - sql = "DELETE FROM ip_net_pool AS po WHERE %s" % where + sql = """LOCK TABLE ip_net_plan IN EXCLUSIVE MODE; +LOCK TABLE ip_net_vrf IN EXCLUSIVE MODE; +LOCK TABLE ip_net_pool IN EXCLUSIVE MODE; +DELETE FROM ip_net_pool AS po WHERE %s""" % where self._execute(sql, params) # write to audit table @@ -3095,7 +3101,10 @@ def _db_remove_prefix(self, spec, recursive = False): else: where, params = self._expand_prefix_spec(spec) - sql = "DELETE FROM ip_net_plan AS p WHERE %s" % where + sql = """LOCK TABLE ip_net_plan IN EXCLUSIVE MODE; +LOCK TABLE ip_net_vrf IN EXCLUSIVE MODE; +LOCK TABLE ip_net_pool IN EXCLUSIVE MODE; +DELETE FROM ip_net_plan AS p WHERE %s""" % where self._execute(sql, params) @@ -3934,7 +3943,10 @@ def remove_asn(self, auth, asn): # remove where, params = self._expand_asn_spec(asn) - sql = "DELETE FROM ip_net_asn WHERE " + where + sql = """LOCK TABLE ip_net_plan IN EXCLUSIVE MODE; +LOCK TABLE ip_net_vrf IN EXCLUSIVE MODE; +LOCK TABLE ip_net_pool IN EXCLUSIVE MODE; +DELETE FROM ip_net_asn WHERE """ + where self._execute(sql, params) # write to audit table