diff --git a/Dockerfile.deploy b/Dockerfile.deploy index 8f7aacd..55166b3 100644 --- a/Dockerfile.deploy +++ b/Dockerfile.deploy @@ -1,5 +1,8 @@ FROM cpantesters/schema +RUN apt-get update && apt-get install -y default-mysql-client COPY etc/docker/deploy-db.sh ./deploy-db.sh COPY etc/docker/mysql.conf ./.cpanstats.cnf +COPY var/schema/metabase-schema.sql ./metabase-schema.sql +COPY var/schema/legacy-schema.sql ./legacy-schema.sql RUN chmod +x deploy-db.sh CMD [ './deploy-db.sh' ] diff --git a/README.md b/README.md index 5339304..d775417 100644 --- a/README.md +++ b/README.md @@ -92,16 +92,16 @@ the database can be reprocessed any number of times. To re-process an existing test report, run the `beam run report process` command inside the `backend` container: - docker-compose exec backend -- beam run report process dc9c7be4-1985-11ea-9825-b8cf277f9bb7 + docker-compose exec backend beam run report process dc9c7be4-1985-11ea-9825-b8cf277f9bb7 To send a request to re-process the report to the Minion job queue, use `beam run report queue`: - docker-compose exec backend -- beam run report queue dc9c7be4-1985-11ea-9825-b8cf277f9bb7 + docker-compose exec backend beam minion run report queue dc9c7be4-1985-11ea-9825-b8cf277f9bb7 To see all the available backend commands: - docker-compose exec backend -- beam list + docker-compose exec backend beam list #### Test reporting clients @@ -142,20 +142,19 @@ this repository. ### Database and Metabase (`cpantesters-schema`) -The primary source of CPANTesters data is the Metabase. This is the -database that the CPANTesters reporters write to. This database is -managed elsewhere. +The primary source of CPANTesters data is the schema. This is the +database that the CPANTesters reporters write to. -The CPANTesters project reads from the Metabase and writes raw data and -report summaries to a local MySQL database. This local MySQL database is -managed here. +A mock version of the previous main database, Metabase, is still written +to for legacy processes. These need to be updated to use the main +database and then the Metabase can be decomissioned. ### Backend ETL (`cpantesters-backend`) -The backend processes read from the Metabase and writes to the local -database. These processes also send out the regular report e-mails, -summarize the raw reports in to easily-queried tables, and maintain the -data the web app requires. +The backend processes turn the full test reports into summaries, +statistics, and metrics. These processes also send out the regular +report e-mails, summarize the raw reports in to easily-queried tables, +and maintain the data the web app requires. ### Web app (`cpantesters-api`, `cpantesters-web`) @@ -163,6 +162,9 @@ The CPANTesters web app is the final piece and uses the MySQL database and the data the Backend ETL provides to make a frontend that users can use. There are also APIs available from the web frontend. +The web app has its own database to manage user accounts and +preferences. + ## Repository Overview ### Rexfile diff --git a/etc/docker/deploy-db.sh b/etc/docker/deploy-db.sh index 00a2649..5a07e44 100644 --- a/etc/docker/deploy-db.sh +++ b/etc/docker/deploy-db.sh @@ -3,3 +3,20 @@ if which cpantesters-web; then ./wait-for-it.sh db_web:3306 -- cpantesters-web schema upgrade fi + +### Create a metabase instance on the testers database. This will be +# needed until all the backend processes are migrated to the +# CPAN::Testers::Backend project and updated to use the primary database. +# Generated with: +# mysqldump --defaults-file=~/.cpanstats.cnf --no-data --databases metabase +# and then edited to add "IF NOT EXISTS" so that it can be run +# repeatedly. This database should not change, because we're trying to +# get rid of it. New features should be added to the testers database, +# not the metabase. +mysql --defaults-file=~/.cpanstats.cnf < ./metabase-schema.sql +# Legacy tables that will need updating for now. +# Generated with: +# mysqldump --defaults-file=~/.cpanstats.cnf --no-data --databases cpanstats --tables page_requests +# and then edited to add "IF NOT EXISTS" so that it can be run +mysql --defaults-file=~/.cpanstats.cnf < ./legacy-schema.sql + diff --git a/var/schema/legacy-schema.sql b/var/schema/legacy-schema.sql new file mode 100644 index 0000000..5672c0b --- /dev/null +++ b/var/schema/legacy-schema.sql @@ -0,0 +1,13 @@ +-- Generated with +-- mysqldump --defaults-file=~/.cpanstats.cnf --no-data --databases cpanstats --tables page_requests +-- And then edited to add "IF NOT EXISTS" to the create table statements +CREATE TABLE IF NOT EXISTS `page_requests` ( + `type` varchar(8) NOT NULL, + `name` varchar(255) NOT NULL, + `weight` int(2) unsigned NOT NULL, + `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `id` int(10) unsigned DEFAULT '0', + KEY `IXNAME` (`name`), + KEY `IXTYPE` (`type`), + KEY `IXID` (`id`) +); diff --git a/var/schema/metabase-schema.sql b/var/schema/metabase-schema.sql new file mode 100644 index 0000000..332dc99 --- /dev/null +++ b/var/schema/metabase-schema.sql @@ -0,0 +1,26 @@ +-- Generated with +-- mysqldump --defaults-file=~/.cpanstats.cnf --no-data --databases metabase +-- And then edited to add "IF NOT EXISTS" to the create table statements + +CREATE DATABASE IF NOT EXISTS `metabase`; +USE `metabase`; + +CREATE TABLE IF NOT EXISTS `metabase` ( + `guid` char(36) NOT NULL, + `id` int(10) unsigned NOT NULL, + `updated` varchar(32) DEFAULT NULL, + `report` longblob NOT NULL, + `fact` longblob, + PRIMARY KEY (`guid`), + KEY `id` (`id`), + KEY `updated` (`updated`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; + +CREATE TABLE IF NOT EXISTS `testers_email` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `resource` varchar(64) NOT NULL, + `fullname` varchar(255) NOT NULL, + `email` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `resource` (`resource`) +) ENGINE=MyISAM AUTO_INCREMENT=10311 DEFAULT CHARSET=latin1;