Merge pull request ckan#8504 from ckan/requirements-upgrade-november-…

…2024

Requirements upgrade November 2024

.circleci/config.yml

@@ -37,7 +37,7 @@ defaults:
 
   redis_image: &redis_image
     # Note, this should be in sync with test-infrastructure/docker-compose.yml
-    image: redis:3
+    image: redis:7
     name: ckan-redis
 
   solr_image: &solr_image

changes/8504.misc

@@ -0,0 +1 @@
+Upgrade requirements to latest versions

ckan/lib/helpers.py

@@ -26,7 +26,9 @@
     Any, Callable, Match, NoReturn, cast, Dict,
     Iterable, Optional, TypeVar, Union)
 
+
 import dominate.tags as dom_tags
+from dominate.util import raw as raw_dom_tags
 from markdown import markdown
 from bleach import clean as bleach_clean, ALLOWED_TAGS, ALLOWED_ATTRIBUTES
 from ckan.common import asbool, config, current_user
@@ -451,7 +453,7 @@ def _url_for_flask(*args: Any, **kw: Any) -> str:
                 for key, val in kw.items():
                     if isinstance(val, (list, tuple)):
                         for value in val:
-                            if value is None:
+                            if value is None or value == {}:
                                 continue
                             query_args.append(
                                 u'{}={}'.format(
@@ -460,14 +462,15 @@ def _url_for_flask(*args: Any, **kw: Any) -> str:
                                 )
                             )
                     else:
-                        if val is None:
+                        if val is None or val == {}:
                             continue
                         query_args.append(
                             u'{}={}'.format(
                                 quote(str(key)),
                                 quote(str(val))
                             )
                         )
+
                 if query_args:
                     my_url += '?'
                 my_url += '&'.join(query_args)
@@ -801,7 +804,7 @@ def link_to(label: Optional[str], url: str, **attrs: Any) -> Markup:
     attrs['href'] = url
     if label == '' or label is None:
         label = url
-    return literal(str(dom_tags.a(label, **attrs)))
+    return literal(str(dom_tags.a(raw_dom_tags(label), **attrs)))
 
 
 @core_helper

ckan/lib/jobs.py

@@ -21,10 +21,8 @@
 
 import logging
 from typing import Any, Union, Callable, Iterable, Optional, cast
-from redis import Redis
 
 import rq
-from rq.connections import push_connection
 from rq.exceptions import NoSuchJobError
 from rq.job import Job
 from rq.utils import ensure_list
@@ -44,17 +42,6 @@
 _queues: dict[str, rq.Queue] = {}
 
 
-def _connect() -> Redis:
-    u'''
-    Connect to Redis and tell RQ about it.
-
-    Workaround for https://github.com/nvie/rq/issues/479.
-    '''
-    conn = connect_to_redis()
-    push_connection(conn)
-    return conn
-
-
 def _get_queue_name_prefix() -> str:
     u'''
     Get the queue name prefix.
@@ -95,7 +82,7 @@ def get_all_queues() -> list[rq.Queue]:
 
     .. seealso:: :py:func:`get_queue`
     '''
-    redis_conn = _connect()
+    redis_conn = connect_to_redis()
     prefix = _get_queue_name_prefix()
     return [q for q in rq.Queue.all(connection=redis_conn) if
             q.name.startswith(prefix)]
@@ -121,7 +108,7 @@ def get_queue(name: str = DEFAULT_QUEUE_NAME) -> rq.Queue:
         return _queues[fullname]
     except KeyError:
         log.debug(u'Initializing background job queue "{}"'.format(name))
-        redis_conn = _connect()
+        redis_conn = connect_to_redis()
         queue = _queues[fullname] = rq.Queue(fullname, connection=redis_conn)
         return queue
 
@@ -191,7 +178,7 @@ def job_from_id(id: str) -> Job:
     :raises KeyError: if no job with that ID exists.
     '''
     try:
-        return Job.fetch(id, connection=_connect())
+        return Job.fetch(id, connection=connect_to_redis())
     except NoSuchJobError:
         raise KeyError(u'There is no job with ID "{}".'.format(id))
 

ckan/tests/lib/test_helpers.py

@@ -169,7 +169,7 @@ def test_debugtoolbar_url(self, ckan_config):
             ({"param": 27.3}, "/dataset/my_dataset?param=27.3"),
             ({"param": True}, "/dataset/my_dataset?param=True"),
             ({"param": None}, "/dataset/my_dataset"),
-            ({"param": {}}, "/dataset/my_dataset?param=%7B%7D"),
+            ({"param": {}}, "/dataset/my_dataset"),
         ],
     )
     def test_url_for_string_route_with_query_param(self, extra, exp):
@@ -326,12 +326,12 @@ class TestHelpersRenderMarkdown(object):
             ),
             (
                 'tag:"test-tag" foobar',
-                '<p><a href="/dataset/?tags=test-tag">tag:&quot;test-tag&quot;</a> foobar</p>',
+                '<p><a href="/dataset/?tags=test-tag">tag:"test-tag"</a> foobar</p>',
                 False,
             ),
             (
                 'tag:"test tag" foobar',
-                '<p><a href="/dataset/?tags=test+tag">tag:&quot;test tag&quot;</a> foobar</p>',
+                '<p><a href="/dataset/?tags=test+tag">tag:"test tag"</a> foobar</p>',
                 False,
             ),
             (
@@ -348,7 +348,7 @@ class TestHelpersRenderMarkdown(object):
             ),
             (
                 'tag:"Test- _." foobar',
-                '<p><a href="/dataset/?tags=Test-+_.">tag:&quot;Test- _.&quot;</a> foobar</p>',
+                '<p><a href="/dataset/?tags=Test-+_.">tag:"Test- _."</a> foobar</p>',
                 False,
             ),
             (
@@ -358,7 +358,7 @@ class TestHelpersRenderMarkdown(object):
             ),
             (
                 u'tag:"Japanese katakana \u30a1" blah',
-                u'<p><a href="/dataset/?tags=Japanese+katakana+%E3%82%A1">tag:&quot;Japanese katakana \u30a1&quot;</a> blah</p>',
+                u'<p><a href="/dataset/?tags=Japanese+katakana+%E3%82%A1">tag:"Japanese katakana \u30a1"</a> blah</p>',
                 False,
             ),
             (
@@ -400,7 +400,7 @@ def test_internal_tag_with_no_closing_quote_does_not_match(self):
     def test_tag_names_match_simple_punctuation(self):
         """Asserts punctuation and capital letters are matched in the tag name"""
         data = 'tag:"Test- _." foobar'
-        output = '<p><a href="/dataset/?tags=Test-+_.">tag:&quot;Test- _.&quot;</a> foobar</p>'
+        output = '<p><a href="/dataset/?tags=Test-+_.">tag:"Test- _."</a> foobar</p>'
         assert h.render_markdown(data) == output
 
     def test_tag_names_do_not_match_commas(self):

ckan/tests/lib/test_jobs.py

@@ -10,6 +10,7 @@
 import rq
 
 import ckan.lib.jobs as jobs
+from ckan.lib.redis import connect_to_redis
 from ckan.common import config
 from ckan.logic import NotFound
 from ckan import model
@@ -102,7 +103,7 @@ def test_foreign_queues_are_ignored(self):
         with changed_config(u"ckan.site_id", u"some-other-ckan-instance"):
             self.enqueue(queue=u"q2")
         # Create queue not related to CKAN
-        rq.Queue(u"q4").enqueue_call(jobs.test_job)
+        rq.Queue(u"q4", connection=connect_to_redis()).enqueue_call(jobs.test_job)
         all_queues = jobs.get_all_queues()
         names = {jobs.remove_queue_name_prefix(q.name) for q in all_queues}
         assert names == {u"q1", u"q2"}

ckan/views/group.py

@@ -617,7 +617,7 @@ def member_dump(id: str, group_type: str, is_organization: bool):
     content_disposition = u'attachment; filename="{name}.csv"'.format(
                                     name=file_name)
     content_type = b'text/csv; charset=utf-8'
-    response.headers['Content-Type'] = content_type  # type: ignore
+    response.headers['Content-Type'] = content_type
     response.headers['Content-Disposition'] = content_disposition
 
     return response

dev-requirements.txt

@@ -3,22 +3,22 @@
 beautifulsoup4==4.12.3
 cookiecutter==2.6.0
 coveralls   #Let Unpinned - Requires latest coveralls
-Faker==26.0.0
-factory-boy==3.3.0
-flask-debugtoolbar==0.15.1
+Faker==30.8.2
+factory-boy==3.3.1
+flask-debugtoolbar==0.16.0
 freezegun==1.5.1
 ipdb==0.13.13
 pip-tools==7.4.1
-Pillow==10.4.0
+Pillow==11.0.0
 responses==0.25.3
-sphinx-rtd-theme==2.0.0
-sphinx==7.3.7
+sphinx-rtd-theme==3.0.1
+sphinx==7.4.7
 toml==0.10.2
-towncrier==23.11.0
+towncrier==24.8.0
 
 pytest==8.2.2
-pytest-cov==5.0.0
+pytest-cov==6.0.0
 pytest-factoryboy==2.7.0
 pytest-freezegun==0.4.2
 pytest-rerunfailures==14.0
-pytest-split==0.9.0
+pytest-split==0.10.0

doc/conf.py

@@ -441,13 +441,11 @@ def write_substitutions_file(**kwargs):
 # Options for HTML output
 # -----------------------
 
-extra_css_files = ['_static/css/custom.css']
 
 on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 if not on_rtd:
     import sphinx_rtd_theme
     html_theme = 'sphinx_rtd_theme'
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 html_sidebars = {
     '**':  ['globaltoc.html'],
@@ -458,7 +456,6 @@ def write_substitutions_file(**kwargs):
     'is_master': is_master,
     'is_supported': is_supported,
     'is_latest_version': is_latest_version,
-    'extra_css_files': extra_css_files,
     'latest_minor_version': latest_minor_version,
 }
 
@@ -488,6 +485,9 @@ def write_substitutions_file(**kwargs):
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
+html_css_files = [
+    'css/custom.css'
+]
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
 #html_last_updated_fmt = '%b %d, %Y'

requirements.in

@@ -1,39 +1,39 @@
 # The file contains the direct ckan requirements (python3).
 # Use pip-compile to create a requirements.txt file from this
-alembic==1.13.2
-Babel==2.15.0
-bleach==6.1.0
+alembic==1.13.3
+Babel==2.16.0
+bleach==6.2.0
 blinker==1.8.2
-certifi>=2024.7.4
+certifi>=2024.8.30
 click==8.1.7
 dominate==2.9.1
 feedgen==1.0.0
 Flask==3.0.3
 Flask-Babel==4.0.0
 Flask-Login==0.6.3
 Flask-Session==0.8.0
-Flask-WTF==1.2.1
+Flask-WTF==1.2.2
 Jinja2==3.1.4
-Markdown==3.6
+Markdown==3.7
 msgspec==0.18.6
 packaging==24.1
 passlib==1.7.4
 polib==1.2.0
 psycopg2==2.9.9
-PyJWT==2.8.0
-pyparsing==3.1.2
+PyJWT==2.9.0
+pyparsing==3.2.0
 python-magic==0.4.27
-pysolr==3.9.0
+pysolr==3.10.0
 python-dateutil==2.9.0.post0
 pytz
 pyyaml==6.0.1
 requests==2.32.3
-rq==1.16.2
-simplejson==3.19.2
-SQLAlchemy[mypy]==1.4.52
-sqlparse==0.5.0
+rq==2.0.0
+simplejson==3.19.3
+SQLAlchemy[mypy]==1.4.54
+sqlparse==0.5.1
 typing_extensions==4.12.2
 tzlocal==5.2
 webassets==2.0
-Werkzeug[watchdog]==3.0.3
-zope.interface==6.4post2
+Werkzeug[watchdog]==3.1.1
+zope.interface==7.1.1