Merge branch 'steemit:master' into bookmarks-for-pr

.gitignore

@@ -125,3 +125,9 @@ tests/failed_blocks/
 /deploy/
 /scripts/hive.sqlite
 .vscode
+
+# docker
+redis_data/*
+postgres_data/*
+!redis_data/.gitkeep
+!postgres_data/.gitkeep

Dockerfile.dev

@@ -0,0 +1,46 @@
+FROM phusion/baseimage:0.11
+
+
+ENV ENVIRONMENT DEV
+ENV LOG_LEVEL INFO
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV PIPENV_VENV_IN_PROJECT 1
+ARG SOURCE_COMMIT
+ENV SOURCE_COMMIT ${SOURCE_COMMIT}
+ARG SCHEMA_HASH
+ENV SCHEMA_HASH adb5cd9b
+#ENV SCHEMA_HASH ${SCHEMA_HASH}
+ARG DOCKER_TAG
+ENV DOCKER_TAG ${DOCKER_TAG}
+
+ENV APP_ROOT /app
+ENV WSGI_APP ${APP_ROOT}/hive/server/serve.py
+ENV HTTP_SERVER_PORT 8080
+
+RUN \
+    apt-get update && \
+    apt-get install -y \
+        awscli \
+        build-essential \
+        daemontools \
+        libffi-dev \
+        libmysqlclient-dev \
+        libssl-dev \
+        make \
+        liblz4-tool \
+        postgresql \
+        postgresql-contrib \
+        python3 \
+        python3-dev \
+        python3-pip \
+        libxml2-dev \
+        libxslt-dev \
+        runit \
+        s3cmd \
+        libpcre3 \
+        libpcre3-dev
+
+RUN \
+    pip3 install --upgrade pip setuptools
+

README.md

@@ -85,6 +85,7 @@ docker logs -f hivemind
 | `HTTP_SERVER_PORT`       | `--http-server-port` | 8080    |
 | `DATABASE_URL`           | `--database-url`     | postgresql://user:pass@localhost:5432/hive |
 | `STEEMD_URL`             | `--steemd-url`       | https://api.steemit.com |
+| `REDIS_URL`              | `--redis-url`        | redis://localhost:6379/ |
 | `MAX_BATCH`              | `--max-batch`        | 50      |
 | `MAX_WORKERS`            | `--max-workers`      | 4       |
 | `TRAIL_BLOCKS`           | `--trail-blocks`     | 2       |

dev_env.sh

@@ -0,0 +1,61 @@
+#/bin/bash
+
+set -xe
+
+container_name=dev-hivemind-env
+
+run() {
+    tag=$1
+    network=$2
+
+    if [ "${tag}" = "" ]; then
+        tag=dev-env
+    fi
+
+    if [ "${network}" = "" ]; then
+        network=bridge
+    fi
+
+    docker run -itd --rm \
+        --name ${container_name} \
+        -v $(pwd):/project \
+        --env-file $(pwd)/.env \
+        --workdir /project \
+        --network ${network} \
+        steemit/hivemind:${tag} \
+        /bin/bash
+}
+
+cli() {
+    docker exec -it ${container_name} /bin/bash
+}
+
+stop() {
+    docker stop ${container_name}
+}
+
+main_func() {
+    op=$1
+    tag=$2
+    network=$3
+
+    case ${op} in
+        run)
+            run $tag $network
+            cli
+            ;;
+        cli)
+            cli
+            ;;
+        stop)
+            stop
+            ;;
+        *)
+            echo "Unknown Command"
+            exit 1
+            ;;
+    esac
+}
+
+main_func $1 $2 $3
+

docker-compose.yml

@@ -6,21 +6,27 @@ services:
       POSTGRES_USER: testuser
       POSTGRES_PASSWORD: testuserpass
       POSTGRES_DB: testdb
-    ports:
-     - 3306:3306
+    volumes:
+      - ./postgres_data:/var/lib/postgresql/data
+    restart: always
+  redis:
+    image: redis
+    volumes:
+      - ./redis_data:/data
+    restart: always
   hive:
     depends_on:
       - db
-    image: steemit/hive
-    build: .
+      - redis
+    image: steemit/hivemind:latest
     environment:
-      DATABASE_URL: postgresql://testuser:testuserpass@db:5432/testdb
       LOG_LEVEL: INFO
+      DATABASE_URL: postgresql://testuser:testuserpass@db:5432/testdb
       STEEMD_URL: https://api.steemit.com
+      REDIS_URL: redis://redis:6379
     links:
       - db:db
+      - redis:redis
     ports:
       - 8080:8080
-    volumes:
-      - .:/app
     restart: always

hive/conf.py

@@ -31,8 +31,9 @@ def init_argparse(cls, strict=True, **kwargs):
         add('mode', nargs='*', default=['sync'])
 
         # common
-        add('--database-url', env_var='DATABASE_URL', required=False, help='database connection url', default='')
-        add('--steemd-url', env_var='STEEMD_URL', required=False, help='steemd/jussi endpoint', default='https://api.steemit.com')
+        add('--database-url', env_var='DATABASE_URL', required=True, help='database connection url', default='')
+        add('--steemd-url', env_var='STEEMD_URL', required=True, help='steemd/jussi endpoint', default='https://api.steemit.com')
+        add('--redis-url', env_var='REDIS_URL', required=True, help='redis connection url', default='')
         add('--muted-accounts-url', env_var='MUTED_ACCOUNTS_URL', required=False, help='url to flat list of muted accounts', default='')
 
         # server

hive/db/db_state.py

@@ -174,7 +174,7 @@ def status():
         row = DbState.db().query_row(sql)
         return dict(db_head_block=row['num'],
                     db_head_time=str(row['created_at']),
-                    db_head_age=int(time.time() - row['ts']))
+                    db_head_age=int(time.time() - float(row['ts'])))
 
     @classmethod
     def _is_schema_loaded(cls):

hive/indexer/accounts.py

@@ -211,7 +211,7 @@ def _sql(cls, account, cached_at):
             'display_name':  profile['name'],
             'about':         profile['about'],
             'location':      profile['location'],
-            'website':       profile['website'],
+            'website':       profile['website'][0:100] if len(profile['website']) > 100 else profile['website'],
             'profile_image': profile['profile_image'],
             'cover_image':   profile['cover_image'],
 

hive/server/common/helpers.py

@@ -20,6 +20,7 @@ async def wrapper(*args, **kwargs):
         try:
             return await function(*args, **kwargs)
         except (ApiError, AssertionError, TypeError, Exception) as e:
+            log.error("ERR-DEBUG: args: {%s}, kwargs: {%s}", args, kwargs)
             if isinstance(e, KeyError):
                 #TODO: KeyError overloaded for method not found. Any KeyErrors
                 #      captured in this decorater are likely irrelevant to

hive/server/condenser_api/cursor.py

@@ -4,9 +4,14 @@
 from dateutil.relativedelta import relativedelta
 
 from hive.utils.normalize import rep_to_raw
+import re
 
 # pylint: disable=too-many-lines
 
+def to_string_without_special_char(v):
+    cleaned_string = re.sub('[^A-Za-z0-9]+', '', str(v))
+    return cleaned_string
+
 def last_month():
     """Get the date 1 month ago."""
     return datetime.now() + relativedelta(months=-1)
@@ -395,17 +400,30 @@ async def pids_by_replies_to_account(db, start_author: str, start_permlink: str
         seek = "AND id <= :start_id"
     else:
         parent_account = start_author
-
+    
     sql = """
-       SELECT id FROM hive_posts
-        WHERE parent_id IN (SELECT id FROM hive_posts
-                             WHERE author = :parent
-                               AND is_deleted = '0'
-                          ORDER BY id DESC
-                             LIMIT 10000) %s
-          AND is_deleted = '0'
-     ORDER BY id DESC
-        LIMIT :limit
-    """ % seek
+    SELECT id FROM hive_posts
+    WHERE author = :parent
+    AND is_deleted = '0'
+    ORDER BY id DESC
+    LIMIT 10000
+    """
+
+    cache_key = "hive_posts-" + parent_account + "-is_deleted_0"
+    id_res = await db.query_all(sql, parent=parent_account, cache_key=cache_key)
+    if id_res == None or len(id_res) == 0:
+        return None
+    tmp_ids = []
+    for el in id_res:
+        tmp_ids.append(str(el[0]))
+    ids = ",".join(tmp_ids)
 
-    return await db.query_col(sql, parent=parent_account, start_id=start_id, limit=limit)
+    sql = """
+    SELECT id FROM hive_posts
+    WHERE parent_id IN (%s) %s
+    AND is_deleted = '0'
+    ORDER BY id DESC
+    LIMIT :limit
+    """ % (ids, seek)
+
+    return await db.query_col(sql, limit=limit)

hive/server/condenser_api/get_state.py

@@ -4,7 +4,6 @@
 import logging
 from collections import OrderedDict
 import ujson as json
-from aiocache import cached
 
 from hive.utils.normalize import legacy_amount
 from hive.server.common.mutes import Mutes
@@ -278,16 +277,24 @@ async def _load_discussion(db, author, permlink):
     # return all nodes keyed by ref
     return {refs[pid]: post for pid, post in posts.items()}
 
-@cached(ttl=1800, timeout=1200)
 async def _get_feed_price(db):
     """Get a steemd-style ratio object representing feed price."""
-    price = await db.query_one("SELECT usd_per_steem FROM hive_state")
+    price = await db.query_one(
+        "SELECT usd_per_steem FROM hive_state",
+        cache_key="_get_feed_price",
+        cache_ttl=3)
     return {"base": "%.3f SBD" % price, "quote": "1.000 STEEM"}
 
-@cached(ttl=1800, timeout=1200)
 async def _get_props_lite(db):
     """Return a minimal version of get_dynamic_global_properties data."""
-    raw = json.loads(await db.query_one("SELECT dgpo FROM hive_state"))
+    tmp = await db.query_one(
+        "SELECT dgpo FROM hive_state",
+        cache_key="_hive_state_dgpo",
+        cache_ttl=3)
+    if tmp is None or tmp == '':
+        return dict()
+
+    raw = json.loads(tmp)
 
     # convert NAI amounts to legacy
     nais = ['virtual_supply', 'current_supply', 'current_sbd_supply',

hive/server/condenser_api/objects.py

@@ -146,7 +146,7 @@ def _condenser_post_object(row, truncate_body=0):
     post['promoted'] = "%.3f SBD" % row['promoted']
 
     post['replies'] = []
-    post['body_length'] = len(row['body'])
+    post['body_length'] = len(row['body']) if row['body'] is not None else ''
     post['active_votes'] = _hydrate_active_votes(row['votes'])
     post['author_reputation'] = rep_to_raw(row['author_rep'])
 

hive/server/condenser_api/tags.py

@@ -1,10 +1,8 @@
 """condenser_api trending tag fetching methods"""
 
-from aiocache import cached
 from hive.server.common.helpers import (return_error_info, valid_tag, valid_limit)
 
 @return_error_info
-@cached(ttl=7200, timeout=1200)
 async def get_top_trending_tags_summary(context):
     """Get top 50 trending tags among pending posts."""
     # Same results, more overhead:
@@ -17,10 +15,9 @@ async def get_top_trending_tags_summary(context):
       ORDER BY SUM(payout) DESC
          LIMIT 50
     """
-    return await context['db'].query_col(sql)
+    return await context['db'].query_col(sql, cache_key='get_top_trending_tags_summary', cache_ttl=5*60)
 
 @return_error_info
-@cached(ttl=3600, timeout=1200)
 async def get_trending_tags(context, start_tag: str = '', limit: int = 250):
     """Get top 250 trending tags among pending posts, with stats."""
 
@@ -51,7 +48,7 @@ async def get_trending_tags(context, start_tag: str = '', limit: int = 250):
     """ % seek
 
     out = []
-    for row in await context['db'].query_all(sql, limit=limit, start_tag=start_tag):
+    for row in await context['db'].query_all(sql, limit=limit, start_tag=start_tag, cache_key="get_trending_tags_"+start_tag+"_"+limit, cache_ttl=5*60):
         out.append({
             'name': row['category'],
             'comments': row['total_posts'] - row['top_posts'],