Skip to content

Commit

Permalink
Perspective view better pagination -- ispras/lingvodoc-react#1133 (#1516
Browse files Browse the repository at this point in the history 
)

* Perspective view better pagination -- ispras/lingvodoc-react#1133 (#1513)
  • Loading branch information
@vmonakhov
vmonakhov authored Sep 4, 2024
1 parent 3203ecb commit 0a3e9b9
Show file tree
Hide file tree
Showing 4 changed files with 376 additions and 139 deletions.
308 changes: 235 additions & 73 deletions lingvodoc/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,14 @@
Index,
literal,
or_,
func,
desc,
Sequence,
Table,
tuple_)

from sqlalchemy.sql.expression import nullsfirst, nullslast

from sqlalchemy.dialects.postgresql import JSONB, UUID

from sqlalchemy.ext.compiler import compiles
Expand Down Expand Up @@ -66,7 +70,7 @@
# Project imports.

import lingvodoc.cache.caching as caching

from pdb import set_trace as A

# Setting up logging.
log = logging.getLogger(__name__)
Expand Down Expand Up @@ -1022,6 +1026,25 @@ class Dictionary(
domain = Column(Integer, default=0)


class PerspectivePage(
CompositeIdMixin,
TableNameMixin,
ParentMixin,
CreatedAtMixin,
TranslationMixin,
StateMixin,
MarkedForDeletionMixin,
AdditionalMetadataMixin,
ObjectTOCMixin,
Base):
"""
This object used to compile and get list of lexical entries with extra information
after filtering, sorting and pagination on backend
Parent: DictionaryPerspective
"""
__parentname__ = 'DictionaryPerspective'


class DictionaryPerspective(
CompositeIdMixin,
TableNameMixin,
Expand Down Expand Up @@ -1899,99 +1922,238 @@ def remove_keys(obj, rubbish):
@classmethod
def graphene_track_multiple(
cls,
lexs,
lexes,
publish = None,
accept = None,
delete = False,
filter = None,
sort_by_field = None,
is_ascending = None,
is_case_sens = True,
is_regexp = False,
created_entries = [],
check_perspective = True):

deleted_per = []
alive_lexes = []

if check_perspective:
deleted_per = DictionaryPerspective.get_deleted()

filtered_lexes = []
for x in lexes:

deleted_persps = DictionaryPerspective.get_deleted()
for i in lexs:
if (i[2], i[3]) not in deleted_persps:
filtered_lexes.append(i)
if len(x) >= 4 and (x[2], x[3]) in deleted_per:
continue

else:
alive_lexes.append({'client_id': x[0], 'object_id': x[1]})

filtered_lexes = lexs
temp_table_name = 'lexical_entries_temp_table' + str(uuid.uuid4()).replace("-", "")

ls = []
DBSession.execute(
'''create TEMPORARY TABLE %s (client_id BIGINT, object_id BIGINT) on COMMIT DROP;''' % temp_table_name)

for i, x in enumerate(filtered_lexes):
ls.append({'traversal_lexical_order': i, 'client_id': x[0], 'object_id': x[1]})
class Tempo(Base):

if not ls:
return []
__tablename__ = temp_table_name
__table_args__ = {'prefixes': ['TEMPORARY']}

pub_filter = ""
client_id = Column(SLBigInteger(), primary_key=True)
object_id = Column(SLBigInteger(), primary_key=True)

if publish is not None or accept is not None or delete is not None:
where_cond = list()
if accept:
where_cond.append("publishingentity.accepted = True")
if accept is False:
where_cond.append("publishingentity.accepted = False")
if publish:
where_cond.append("publishingentity.published = True")
if publish is False:
where_cond.append("publishingentity.published = False")
if delete:
where_cond.append("cte_expr.marked_for_deletion = True")
if delete is False:
where_cond.append("cte_expr.marked_for_deletion = False")
where_cond = ["WHERE", " AND ".join(where_cond)]
pub_filter = " ".join(where_cond)
if alive_lexes:
DBSession.execute(
Tempo.__table__
.insert()
.values(alive_lexes))

temp_table_name = 'lexical_entries_temp_table' + str(uuid.uuid4()).replace("-", "")
# We need just lexical entry and entity id and entity's content for sorting and filtering

DBSession.execute(
'''create TEMPORARY TABLE %s (traversal_lexical_order INTEGER, client_id BIGINT, object_id BIGINT) on COMMIT DROP;''' % temp_table_name)
entities_query = (
DBSession
.query(
Entity.client_id,
Entity.object_id,
Entity.parent_client_id,
Entity.parent_object_id,
Entity.content)

DBSession.execute(
'''insert into %s (traversal_lexical_order, client_id, object_id) values (:traversal_lexical_order, :client_id, :object_id);''' % temp_table_name,
ls)
.filter(
Entity.parent_client_id == Tempo.client_id,
Entity.parent_object_id == Tempo.object_id))

statement = text('''
WITH cte_expr AS
(SELECT
entity.*,
{0}.traversal_lexical_order AS traversal_lexical_order
FROM entity
INNER JOIN {0}
ON
entity.parent_client_id = {0}.client_id
AND entity.parent_object_id = {0}.object_id
)
SELECT
cte_expr.client_id,
cte_expr.object_id,
cte_expr.parent_client_id,
cte_expr.parent_object_id,
cte_expr.self_client_id,
cte_expr.self_object_id,
cte_expr.link_client_id,
cte_expr.link_object_id,
cte_expr.field_client_id,
cte_expr.field_object_id,
cte_expr.locale_id,
cte_expr.marked_for_deletion,
cte_expr.content,
cte_expr.additional_metadata,
cte_expr.created_at,
publishingentity.*
FROM cte_expr
LEFT JOIN publishingentity
ON publishingentity.client_id = cte_expr.client_id AND publishingentity.object_id = cte_expr.object_id
{1}
ORDER BY cte_expr.traversal_lexical_order;
'''.format(temp_table_name, pub_filter))
filed_lexes = entities_query.with_entities('parent_client_id', 'parent_object_id')

entries = DBSession.query(Entity, PublishingEntity).from_statement(statement) .options(joinedload('publishingentity')).yield_per(100)
# Collect all empty lexes including created ones

return entries
empty_lexes = (
DBSession
.query(
Tempo.client_id,
Tempo.object_id)

.filter(
tuple_(Tempo.client_id, Tempo.object_id)
.notin_(filed_lexes))

.all())

# Apply user's custom filter

if filter:

# Filter from special fields
filtered_entities = entities_query.filter(
Entity.field_id != (66, 25))

if is_regexp:
if is_case_sens:
filtered_entities = filtered_entities.filter(
Entity.content.op('~')(filter)).cte()
else:
filtered_entities = filtered_entities.filter(
Entity.content.op('~*')(filter)).cte()
else:
if is_case_sens:
filtered_entities = filtered_entities.filter(
Entity.content.like(f"%{filter}%")).cte()
else:
filtered_entities = filtered_entities.filter(
Entity.content.ilike(f"%{filter}%")).cte()

entities_query = entities_query.filter(
Entity.parent_client_id == filtered_entities.c.parent_client_id,
Entity.parent_object_id == filtered_entities.c.parent_object_id)

entities_cte = entities_query.cte()

# Create sorting_cte to order by it

sorting_cte = None

if sort_by_field:

field_entities = entities_query.filter(Entity.field_id == sort_by_field).cte()

alpha_entities = (
DBSession
.query(
field_entities.c.parent_client_id.label('lex_client_id'),
field_entities.c.parent_object_id.label('lex_object_id'),
func.min(func.lower(field_entities.c.content)).label('first_entity'),
func.max(func.lower(field_entities.c.content)).label('last_entity'),
func.count().label('count_entity'))

#.filter(func.length(field_entities.c.content) > 0)

.group_by('lex_client_id', 'lex_object_id')

.cte()
)

sorting_cte = (
DBSession
.query(
entities_cte.c.parent_client_id,
entities_cte.c.parent_object_id,
entities_cte.c.client_id,
entities_cte.c.object_id,
alpha_entities.c.first_entity,
alpha_entities.c.last_entity,
alpha_entities.c.count_entity,
field_entities.c.content.label('order_content'))

.outerjoin(
alpha_entities, and_(
alpha_entities.c.lex_client_id == entities_cte.c.parent_client_id,
alpha_entities.c.lex_object_id == entities_cte.c.parent_object_id))

.outerjoin(
field_entities, and_(
field_entities.c.client_id == entities_cte.c.client_id,
field_entities.c.object_id == entities_cte.c.object_id))

.cte())

entities_cte = sorting_cte

# Finally, filter and sort Entity and PublishingEntity objects

entities_result = (
DBSession
.query(
Entity,
PublishingEntity)

.outerjoin(
PublishingEntity))

# Pre-filtering

if accept is not None:
entities_result = entities_result.filter(PublishingEntity.accepted == accept)
if publish is not None:
entities_result = entities_result.filter(PublishingEntity.published == publish)
if delete is not None:
entities_result = entities_result.filter(Entity.marked_for_deletion == delete)

# Get new entities from entities_before_custom_filtering

new_entities_result = (
entities_result
.filter(
tuple_(Entity.parent_client_id, Entity.parent_object_id)
.in_(created_entries)))

# Filter and join at once to get and sort old entities

old_entities_result = (
entities_result
.filter(
entities_cte.c.client_id == Entity.client_id,
entities_cte.c.object_id == Entity.object_id,

tuple_(Entity.parent_client_id, Entity.parent_object_id)
.notin_(created_entries)))

# Custom sorting

if sorting_cte is not None:

if is_ascending:

old_entities_result = old_entities_result.order_by(
entities_cte.c.first_entity,
nullsfirst(entities_cte.c.count_entity), # for 'Paradigm and contexts' field
entities_cte.c.parent_client_id,
entities_cte.c.parent_object_id,
func.lower(entities_cte.c.order_content)
)

else:

old_entities_result = old_entities_result.order_by(
desc(entities_cte.c.last_entity),
nullslast(entities_cte.c.count_entity.desc()), # for 'Paradigm and contexts' field
entities_cte.c.parent_client_id,
entities_cte.c.parent_object_id,
desc(func.lower(entities_cte.c.order_content))
)

# Default sorting

old_entities_result = old_entities_result.order_by(
Entity.parent_client_id,
Entity.parent_object_id,
Entity.client_id,
Entity.object_id)

return (
new_entities_result,
old_entities_result
.options(
joinedload('publishingentity'))
.yield_per(100),
empty_lexes)


class Entity(
Expand Down
Loading

0 comments on commit 0a3e9b9

Please sign in to comment.