diff --git a/app/tasks/crawl.py b/app/tasks/crawl.py index 0029668..a1dc10f 100644 --- a/app/tasks/crawl.py +++ b/app/tasks/crawl.py @@ -23,12 +23,8 @@ git_binary = path break -args = [git_binary, 'pull', 'origin', 'master'] -proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, cwd=dirname) -output, _ = proc.communicate() -print(output.decode(encoding='utf-8')) - +args = [git_binary, 'pull', '--ff-only', 'origin', 'master'] +print(subprocess.check_output(args, cwd=dirname, encoding='utf-8')) if explicit_package: valid_sources = package.sources.sources_for(explicit_package) diff --git a/dev-servers.sh b/dev-servers.sh index 5ef26ca..f7ee5bd 100755 --- a/dev-servers.sh +++ b/dev-servers.sh @@ -17,11 +17,32 @@ mkdir -p "$PROJ_DIR/data/pgsql" mkdir -p "$PROJ_DIR/data/redis" mkdir -p "$PROJ_DIR/data/logs" -if [[ ! -d "$PROJ_DIR/data/pgsql/base" ]]; then - initdb "$PROJ_DIR/data/pgsql" - createdb -U postgres -E 'UTF-8' package_control +# Create run directory with elevated privileges +PGSQL_RUN_DIR=/var/run/postgresql +if [[ ! -d "$PGSQL_RUN_DIR" ]]; then + sudo mkdir -p "$PGSQL_RUN_DIR" + sudo chown $(whoami) "$PGSQL_RUN_DIR" +fi + +HAS_DB=1 +if [[ ! -e "$PROJ_DIR/data/pgsql/PG_VERSION" ]]; then + HAS_DB=0 + initdb -U postgres -E UTF-8 -D "$PROJ_DIR/data/pgsql" + sleep 1 fi pg_ctl -D "$PROJ_DIR/data/pgsql" -l "$PROJ_DIR/data/logs/pgsql.log" start -redis-server --port 6379 + +if ((!HAS_DB)); then + createdb -U postgres -E UTF-8 package_control + psql -U postgres -d package_control -f "$PROJ_DIR/setup/sql/up.sql" +fi + +redis-server --port 6379 --daemonize yes --loglevel warning --logfile $PROJ_DIR/data/logs/redis.log + +# enter sql shell +psql -U postgres package_control + +# shutdown servers +redis-cli -h 127.0.0.1 -p 6379 shutdown pg_ctl -D "$PROJ_DIR/data/pgsql" stop diff --git a/dev.sh b/dev.sh index a83ab65..a0808b6 100755 --- a/dev.sh +++ b/dev.sh @@ -1,4 +1,4 @@ -#/usr/bin/env bash +#!/usr/bin/env bash tmux has-session -t packagecontrol if [[ $? != 0 ]]; then @@ -12,7 +12,7 @@ if [[ $? != 0 ]]; then tmux select-layout -t packagecontrol tiled sleep 0.6 - tmux send-keys -t packagecontrol:0.0 'psql -U postgres package_control' C-m + tmux send-keys -t packagecontrol:0.0 './dev-servers.sh' C-m tmux send-keys -t packagecontrol:0.1 '. venv/bin/activate' C-m tmux send-keys -t packagecontrol:0.2 '. venv/bin/activate' C-m tmux send-keys -t packagecontrol:0.3 '. venv/bin/activate' C-m diff --git a/secrets-example.yml b/secrets-example.yml index 4617da9..dcd908e 100644 --- a/secrets-example.yml +++ b/secrets-example.yml @@ -1,3 +1,6 @@ +bitbucket_access_token_wbond: "Enter your personal access token here" +github_access_token_packagecontrol-bot: "Enter your personal access token here" +gitlab_access_token_wbond: "Enter your personal access token here" github_client_id: "Visit https://github.com/settings/applications to register an application" github_client_secret: "Visit https://github.com/settings/applications to register an application" rollbar_key: "Visit https://rollbar.com/signup/ and sign up for a free tier account" diff --git a/servers.sh b/servers.sh index 573478b..0e2ea64 100755 --- a/servers.sh +++ b/servers.sh @@ -35,7 +35,7 @@ if [[ $TASK == "start" ]]; then if [[ ! -f "$PROJ_DIR/data/ssl/dhparam.pem" ]]; then echo -n "Generating dhparam for TLS ..." - openssl dhparam -out $PROJ_DIR/data/ssl/dhparam.pem 2048 + openssl dhparam -out $PROJ_DIR/data/ssl/dhparam.pem 2048 echo " done" fi @@ -97,17 +97,17 @@ if [[ $TASK == "start" ]]; then listen 443 ssl http2; server_name dev.packagecontrol.io; - ssl_certificate $PROJ_DIR/data/ssl/fullchain.pem; - ssl_certificate_key $PROJ_DIR/data/ssl/privkey.pem; - ssl_dhparam $PROJ_DIR/data/ssl/dhparam.pem; - ssl_protocols TLSv1.2 TLSv1.1 TLSv1; - ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS'; - ssl_prefer_server_ciphers on; - ssl_session_timeout 5m; - - ssl_stapling on; - ssl_stapling_verify on; - ssl_trusted_certificate $PROJ_DIR/data/ssl/fullchain.pem; + ssl_certificate $PROJ_DIR/data/ssl/fullchain.pem; + ssl_certificate_key $PROJ_DIR/data/ssl/privkey.pem; + ssl_dhparam $PROJ_DIR/data/ssl/dhparam.pem; + ssl_protocols TLSv1.2 TLSv1.1 TLSv1; + ssl_ciphers 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS'; + ssl_prefer_server_ciphers on; + ssl_session_timeout 5m; + + ssl_stapling on; + ssl_stapling_verify on; + ssl_trusted_certificate $PROJ_DIR/data/ssl/fullchain.pem; root $PROJ_DIR/public;