Internal reports (#19)

* add internal reports functionality

* make singular

* fix tests, and bump rest-tools version

* bump python version and github CI versions

* stop being fancy with coverage html

.github/workflows/docker_build.yml

@@ -12,24 +12,17 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       -
         name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v1
+        uses: docker/metadata-action@v3
         with:
           images: wipac/${{ github.event.repository.name }}
           tag-semver: |
             {{version}}
             {{major}}.{{minor}}
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      -
-        name: Login to DockerHub
+      - name: Login to DockerHub
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v1
         with:

.github/workflows/pr.yml

@@ -11,11 +11,11 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
@@ -33,26 +33,6 @@ jobs:
     - name: Coverage
       run: |
         coverage run -m pytest tests
-    - name: Coverage html
-      run: |
-        coverage html
-        cd htmlcov && tar -zcf ../htmlcov.tar.gz *
-    - name: Upload html coverage results
-      env:
-        USER: ${{ secrets.DOCS_USERNAME }}
-        PASS: ${{ secrets.DOCS_PASSWORD }}
-      run: |
-        curl --data-binary "@htmlcov.tar.gz" -XPUT "https://$USER:[email protected]/api/upload?path=$GITHUB_REPOSITORY/${{ github.head_ref }}/${{ github.sha }}"
-    - uses: actions/github-script@v3
-      with:
-        github-token: ${{secrets.GITHUB_TOKEN}}
-        script: |
-          github.issues.createComment({
-            issue_number: context.issue.number,
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            body: "Coverage report available on https://docs.icecube.aq/" + context.repo.owner + "/" + context.repo.repo + "/${{ github.head_ref }}/${{ github.sha }}"
-          })
     - name: Coverage Text Report
       run: |
         coverage report -m --fail-under=50

.github/workflows/python-package.yml

@@ -13,11 +13,11 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8
+FROM python:3.9
 
 RUN useradd -m -U app
 

pubs/__init__.py

@@ -13,6 +13,7 @@
     "proceeding": "Conference Proceeding",
     "thesis": "Thesis",
     "other": "Other",
+    "internal": "Internal Report",
 }
 
 PROJECTS = {

pubs/server.py

@@ -13,7 +13,8 @@
 import itertools
 
 from tornado.web import RequestHandler, HTTPError
-from rest_tools.server import RestServer, from_environment, catch_error
+from rest_tools.server import RestServer, catch_error
+from wipac_dev_tools import from_environment
 import motor.motor_asyncio
 import pymongo
 from bson.objectid import ObjectId
@@ -216,6 +217,7 @@ async def post(self):
                     doc = {
                         'title': self.get_argument('new_title').strip(),
                         'authors': [a.strip() for a in self.get_argument('new_authors').split('\n') if a.strip()],
+                        'abstract': self.get_argument('new_abstract', ''),
                         'date': self.get_argument('new_date'),
                         'pub_type': self.get_argument('new_type'),
                         'citation': self.get_argument('new_citation').strip(),
@@ -229,6 +231,7 @@ async def post(self):
                     doc = {
                         'title': self.get_argument('new_title').strip(),
                         'authors': [a.strip() for a in self.get_argument('new_authors').split('\n') if a.strip()],
+                        'abstract': self.get_argument('new_abstract', ''),
                         'date': self.get_argument('new_date'),
                         'pub_type': self.get_argument('new_type'),
                         'citation': self.get_argument('new_citation').strip(),
@@ -320,7 +323,7 @@ def create_server():
     create_indexes(db_url, db_name)
 
     users = {v.split(':')[0]: v.split(':')[1] for v in config['BASIC_AUTH'].split(',') if v}
-    logging.info(f'BASIC_AUTH users: {users.keys()}')
+    logging.info(f'BASIC_AUTH users: {list(users.keys())}')
 
     main_args = {
         'debug': config['DEBUG'],

pubs/static/external.js

@@ -144,6 +144,7 @@ async function Pubs(id, baseurl = 'https://publications.icecube.aq', filters = {
     props: {
       title: String,
       authors: String,
+      abstract: String,
       type: String,
       citation: String,
       date: String,
@@ -181,6 +182,7 @@ async function Pubs(id, baseurl = 'https://publications.icecube.aq', filters = {
   <div><span class="type">({{ type }})</span>
   <span class="citation">{{ citation }}</span>
   <span v-if="show_date" class="date">{{ day_month_year }}</span></div>
+  <div class="abstract_div" v-if="abstract">Abstract: <span class="abstract">{{ abstract }}</span></div>
   <div>
     <span class="downloads" v-if="downloads">Download:
       <span class="download" v-for="link in downloads"><a :href="link" target="_blank">{{ getDomain(link) }}</a></span>

pubs/templates/main.html

@@ -43,6 +43,9 @@ <h2>Selected Publications:</h2>
   <div><span class="type">({{ PUBLICATION_TYPES[pub['type']] }})</span>
   <span class="citation">{{ pub['citation'] }}</span>
   <span class="date">{{ date_format(pub['date']) }}</span></div>
+  {% if pub.get('abstract', '') %}
+  <div class="abstract_div">Abstract: <span class="abstract">{{ pub['abstract'] }}</span></div>
+  {% end %}
   <div>
     {% if pub['downloads'] %}
     <span class="downloads">Download:

pubs/templates/manage.html

@@ -22,6 +22,7 @@ <h2>New Publication</h2>
       <option value="{{ t }}">{{ PUBLICATION_TYPES[t] }}</option>
       {% end %}
     </select></div>
+    <div class="input"><label for="new_abstract">Abstract: </label><textarea name="new_abstract"></textarea></div>
     <div class="input"><label for="new_citation">Citation:</label><textarea name="new_citation"></textarea></div>
     <div class="input"><label for="new_downloads">Download links:</label><textarea name="new_downloads"></textarea></div>
     <div class="input"><label for="new_projects">Project: </label><div class="projects_box">
@@ -101,6 +102,9 @@ <h2>Existing Publications:</h2>
   <div><span class="type">({{ PUBLICATION_TYPES[pub['type']] }})</span>
   <span class="citation">{{ pub['citation'] }}</span>
   <span class="date">{{ date_format(pub['date']) }}</span></div>
+  {% if pub.get('abstract', '') %}
+  <div class="abstract_div">Abstract: <span class="abstract">{{ pub['abstract'] }}</span></div>
+  {% end %}
   <div>
     {% if pub['downloads'] %}
     <span class="downloads">Download:
@@ -189,8 +193,11 @@ <h2>Existing Publications:</h2>
   }
   const pub_id = pub.find('input[name="pub_id"]').val();
 
+  const abstract_text = Array.from(pub.find('.abstract').map(function(){return $(this).text()})).join('');
+
   console.log('title:'+title);
   console.log('authors:');console.log(authors);
+  console.log('abstract:');console.log(abstract_text);
   console.log('pub_type:'+pub_type);
   console.log('citation:'+citation);
   console.log('date:'+date);
@@ -203,6 +210,7 @@ <h2>Existing Publications:</h2>
   $('.new textarea[name="new_authors"]').val(authors.join('\n'));
   $('.new input[name="new_date"]').val(date);
   $('.new select[name="new_type"]').val(pub_type);
+  $('.new textarea[name="new_abstract"]').val(abstract_text);
   $('.new textarea[name="new_citation"]').val(citation);
   $('.new textarea[name="new_downloads"]').val(downloads.join('\n'));
   $('.new input[name="new_projects"]').val(projects);

pubs/utils.py

@@ -37,12 +37,13 @@ def create_indexes(db_url, db_name, background=True):
                                      weights={'title': 10, 'authors': 5, 'citation': 1},
                                      name='text_index', background=background)
 
-def validate(title, authors, pub_type, citation, date, downloads, projects, sites):
+def validate(title, authors, pub_type, abstract, citation, date, downloads, projects, sites):
     assert isinstance(title, str)
     assert isinstance(authors, list)
     for a in authors:
         assert isinstance(a, str)
     assert pub_type in PUBLICATION_TYPES
+    assert isinstance(abstract, str)
     assert isinstance(citation, str)
     assert isinstance(date, str)
     date_format(date)
@@ -56,23 +57,24 @@ def validate(title, authors, pub_type, citation, date, downloads, projects, site
     for s in sites:
         assert s in SITES
 
-async def add_pub(db, title, authors, pub_type, citation, date, downloads, projects, sites=None):
+async def add_pub(db, title, authors, pub_type, abstract, citation, date, downloads, projects, sites=None):
     if not sites:
         sites = []
-    validate(title, authors, pub_type, citation, date, downloads, projects, sites)
+    validate(title, authors, pub_type, abstract, citation, date, downloads, projects, sites)
     data = {
         "title": title,
         "authors": authors,
         "type": pub_type,
         "citation": citation,
         "date": date,
+        "abstract": abstract,
         "downloads": downloads,
         "projects": projects,
         "sites": sites,
     }
     await db.publications.insert_one(data)
 
-async def edit_pub(db, mongo_id, title=None, authors=None, pub_type=None, citation=None, date=None, downloads=None, projects=None, sites=None):
+async def edit_pub(db, mongo_id, title=None, authors=None, pub_type=None, abstract=None, citation=None, date=None, downloads=None, projects=None, sites=None):
     match = {'_id': ObjectId(mongo_id)}
     update = {}
     if title:
@@ -86,6 +88,9 @@ async def edit_pub(db, mongo_id, title=None, authors=None, pub_type=None, citati
     if pub_type:
         assert pub_type in PUBLICATION_TYPES
         update['type'] = pub_type
+    if abstract:
+        assert isinstance(abstract, str)
+        update['abstract'] = abstract
     if citation:
         assert isinstance(citation, str)
         update['citation'] = citation
@@ -139,7 +144,7 @@ def parse_csv(row):
         if isinstance(p['authors'], str):
             p['authors'] = [p['authors']]
         try:
-            validate(p['title'], p['authors'], p['type'], p['citation'], p['date'], p['downloads'], p['projects'], p['sites'])
+            validate(p['title'], p['authors'], p['type'], p.get('abstract', ''), p['citation'], p['date'], p['downloads'], p['projects'], p['sites'])
         except AssertionError:
             raise Exception(f'Error validating pub with title {p["title"][:100]}')
 

requirements.txt

@@ -10,4 +10,4 @@ motor
 pymysql
 unidecode
 beautifulsoup4
--e git+https://github.com/WIPACrepo/rest-tools@v0.2.0#egg=rest_tools
+wipac-rest-tools