From bb302882524d654bd8621d97723ef09930372b14 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Mon, 28 Jul 2014 11:23:33 +1200 Subject: [PATCH 01/11] NEW: Adding report link to setup new queued job --- _config/routes.yml | 7 ++ code/controllers/CMSExternalLinks.php | 20 +++++ code/jobs/CheckExternalLinksJob.php | 4 +- code/reports/BrokenExternalLinksReport.php | 87 ++++++++++++++++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 _config/routes.yml create mode 100644 code/controllers/CMSExternalLinks.php create mode 100644 code/reports/BrokenExternalLinksReport.php diff --git a/_config/routes.yml b/_config/routes.yml new file mode 100644 index 0000000..2284d3a --- /dev/null +++ b/_config/routes.yml @@ -0,0 +1,7 @@ +--- +Name: externallink +After: framework/routes +--- +Director: + rules: + 'admin/externallinks//$Action': 'CMSExternalLinks_Controller' diff --git a/code/controllers/CMSExternalLinks.php b/code/controllers/CMSExternalLinks.php new file mode 100644 index 0000000..e6769d1 --- /dev/null +++ b/code/controllers/CMSExternalLinks.php @@ -0,0 +1,20 @@ +queueJob($externalLinks); + + // redirect to the jobs page + $admin = QueuedJobsAdmin::create(); + $this->Redirect($admin->Link()); + } +} diff --git a/code/jobs/CheckExternalLinksJob.php b/code/jobs/CheckExternalLinksJob.php index 381df5b..ba569ca 100644 --- a/code/jobs/CheckExternalLinksJob.php +++ b/code/jobs/CheckExternalLinksJob.php @@ -1,5 +1,7 @@ 'Checked', + 'Link' => 'External Link', + 'HTTPCode' => 'HTTP Error Code', + 'PageLink' => array( + 'title' => 'Page link is on', + 'link' => true + ), + ); + + public function init() { + parent::init(); + } + + /** + * Returns the report title + * + * @return string + */ + public function title() { + return _t('ExternalBrokenLinksReport.EXTERNALBROKENLINKS',"External broken links report"); + } + + /** + * Returns the column names of the report + * + * @return array + */ + public function columns() { + return self::$columns; + } + + /** + * Alias of columns(), to support the export to csv action + * in {@link GridFieldExportButton} generateExportFileData method. + * @return array + */ + public function getColumns() { + return $this->columns(); + } + + public function sourceRecords() { + $returnSet = new ArrayList(); + $links = BrokenExternalLinks::get(); + foreach ($links as $link) { + $link->PageLink = $link->Page()->Title; + $returnSet->push($link); + } + return $returnSet; + } + + public function getCMSFields() { + $fields = parent::getCMSFields(); + if (class_exists('AbstractQueuedJob')) { + $button = ''; + $runReportButton = new LiteralField( + 'runReport', + sprintf( + $button, + 'admin/externallinks/createQueuedReport', + _t('ExternalBrokenLinksReport.RUNREPORT', 'Create new report') + ) + ); + $fields->push($runReportButton); + $reportResultSpan = ''; + $reportResult = new LiteralField('ResultTitle', $reportResultSpan); + $fields->push($reportResult); + } + return $fields; + } +} From d25adca17506b326c0aef69ba993201bee092ba2 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Mon, 28 Jul 2014 12:39:19 +1200 Subject: [PATCH 02/11] NEW: Adding tests and code tidyup --- README.md | 10 +++--- ...ternalLinks.php => BrokenExternalLink.php} | 7 +++- code/reports/BrokenExternalLinksReport.php | 2 +- code/tasks/CheckExternalLinks.php | 6 ++-- tests/ExternalLinksTest.php | 34 +++++++++++++++++++ tests/ExternalLinksTest.yml | 7 ++++ 6 files changed, 58 insertions(+), 8 deletions(-) rename code/model/{BrokenExternalLinks.php => BrokenExternalLink.php} (66%) create mode 100644 tests/ExternalLinksTest.php create mode 100644 tests/ExternalLinksTest.yml diff --git a/README.md b/README.md index 408a914..ffb35a3 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,10 @@ The external links module is a task and ModelAdmin to track and to report on bro Run the following task *http://path.to.silverstripe/dev/tasks/CheckExternalLinks* to check your site for external broken links. If you have the queuedjobs module installed you can set the task to be run every so ofter -Add the following code to the mysite config to run the job every 24 hours (86400 seconds) - -`Config::inst()->update('CheckExternalLinks', 'QueuedJob', 86400);` - +Add the following yml config to config.yml in mysite/_config have the the task run once every day (86400 seconds) +`--- +Name: externallinkssettings +--- +CheckExternalLinks: + Delay: 86400` diff --git a/code/model/BrokenExternalLinks.php b/code/model/BrokenExternalLink.php similarity index 66% rename from code/model/BrokenExternalLinks.php rename to code/model/BrokenExternalLink.php index e002405..1de3554 100644 --- a/code/model/BrokenExternalLinks.php +++ b/code/model/BrokenExternalLink.php @@ -1,6 +1,6 @@ 'Varchar(2083)', // 2083 is the maximum length of a URL in Internet Explorer. @@ -25,4 +25,9 @@ function canEdit($member = false) { return false; } + function canView($member = false) { + $member = $member ? $member : Member::currentUser(); + $codes = array('content-authors', 'administrators'); + return Permission::checkMember($member, $codes); + } } diff --git a/code/reports/BrokenExternalLinksReport.php b/code/reports/BrokenExternalLinksReport.php index 5a7e79d..4bfe568 100644 --- a/code/reports/BrokenExternalLinksReport.php +++ b/code/reports/BrokenExternalLinksReport.php @@ -57,7 +57,7 @@ public function getColumns() { public function sourceRecords() { $returnSet = new ArrayList(); - $links = BrokenExternalLinks::get(); + $links = BrokenExternalLink::get(); foreach ($links as $link) { $link->PageLink = $link->Page()->Title; $returnSet->push($link); diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index ba2f148..a6ef0b0 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -54,13 +54,15 @@ function run($request) { if($href && function_exists('curl_init')) { $handle = curl_init($href); curl_setopt($handle, CURLOPT_RETURNTRANSFER, TRUE); + curl_setopt($handle, CURLOPT_CONNECTTIMEOUT, 5); + curl_setopt($handle, CURLOPT_TIMEOUT, 10); $response = curl_exec($handle); $httpCode = curl_getinfo($handle, CURLINFO_HTTP_CODE); curl_close($handle); if (($httpCode < 200 || $httpCode > 302) || ($href == '' || $href[0] == '/')) { - $brokenLink = new BrokenExternalLinks(); + $brokenLink = new BrokenExternalLink(); $brokenLink->PageID = $page->ID; $brokenLink->Link = $href; $brokenLink->HTTPCode = $httpCode; @@ -94,7 +96,7 @@ function run($request) { } // run this again if queued jobs exists and is a valid int - $queuedJob = Config::inst()->get('CheckExternalLinks', 'QueuedJob'); + $queuedJob = Config::inst()->get('CheckExternalLinks', 'Delay'); if (isset($queuedJob) && is_int($queuedJob) && class_exists('QueuedJobService')) { $checkLinks = new CheckExternalLinksJob(); singleton('QueuedJobService') diff --git a/tests/ExternalLinksTest.php b/tests/ExternalLinksTest.php new file mode 100644 index 0000000..ae9d898 --- /dev/null +++ b/tests/ExternalLinksTest.php @@ -0,0 +1,34 @@ +objFromFixture('Page', 'working'); + $task = new CheckExternalLinks(); + $task->run($page); + $brokenLinks = BrokenExternalLinks::get(); + $this->assertEquals(0, $brokenLinks->count()); + } + + public function testBrokenLink() { + // uses http://192.0.2.1 for a broken link + $page = $this->objFromFixture('Page', 'broken'); + $task = new CheckExternalLinks(); + $task->run($page); + $brokenLinks = BrokenExternalLinks::get(); + $this->assertEquals(1, $brokenLinks->count()); + } + + public function testReportExists() { + $reports = SS_Report::get_reports(); + $reportNames = array(); + foreach($reports as $report) { + $reportNames[] = $report->class; + } + $this->assertContains('BrokenExternalLinksReport',$reportNames, + 'BrokenExternalLinksReport is in reports list'); + } +} diff --git a/tests/ExternalLinksTest.yml b/tests/ExternalLinksTest.yml new file mode 100644 index 0000000..8a25106 --- /dev/null +++ b/tests/ExternalLinksTest.yml @@ -0,0 +1,7 @@ +Page: + working: + Title: Working Link + Content: 'Localhost' + broken: + Title: Broken Link + Content: 'Broken' \ No newline at end of file From 093322fcd35b3506f358f9a0412899bdc65947e4 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Wed, 30 Jul 2014 12:34:39 +1200 Subject: [PATCH 03/11] NEW: Adding a option to run a batch job to the front end --- README.md | 15 +++-- code/controllers/CMSExternalLinks.php | 70 ++++++++++++++++++---- code/jobs/CheckExternalLinksJob.php | 3 +- code/model/BrokenExternalLink.php | 10 ++++ code/reports/BrokenExternalLinksReport.php | 7 ++- code/tasks/CheckExternalLinks.php | 30 ++++++---- javascript/BrokenExternalLinksReport.js | 45 ++++++++++++++ 7 files changed, 151 insertions(+), 29 deletions(-) create mode 100644 javascript/BrokenExternalLinksReport.js diff --git a/README.md b/README.md index ffb35a3..6999e68 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,22 @@ The external links module is a task and ModelAdmin to track and to report on bro 5. Run in your browser - `/dev/build` to rebuild the database. 6. Run the following task *http://path.to.silverstripe/dev/tasks/CheckExternalLinks* to check for broken external links +## Report ## + +A new report is added called 'External Broken links report' from here you can also start a new job which is run +via AJAX and in batches of 10 so it can be run via content editors who do not have access to jobs or tasks. + ## Dev task ## Run the following task *http://path.to.silverstripe/dev/tasks/CheckExternalLinks* to check your site for external broken links. + +## Queued job ## + If you have the queuedjobs module installed you can set the task to be run every so ofter Add the following yml config to config.yml in mysite/_config have the the task run once every day (86400 seconds) -`--- -Name: externallinkssettings ---- +``` CheckExternalLinks: - Delay: 86400` + Delay: 86400 +``` diff --git a/code/controllers/CMSExternalLinks.php b/code/controllers/CMSExternalLinks.php index e6769d1..5c23815 100644 --- a/code/controllers/CMSExternalLinks.php +++ b/code/controllers/CMSExternalLinks.php @@ -2,19 +2,69 @@ class CMSExternalLinks_Controller extends Controller { - private static $allowed_actions = array('createQueuedReport'); + private static $allowed_actions = array('getJobStatus', 'clear', 'start'); + /* + * Respond to Ajax requests for info on a running job + * also calls continueJob and clear depending on the status of the job + * + * @return string JSON string detailing status of the job + */ + public function getJobStatus() { + $trackID = Session::get('ExternalLinksTrackID'); + if (!$trackID) return; + $noPages = Versioned::get_by_stage('SiteTree', 'Live')->count(); + $result = BrokenExternalPageTrack::get() + ->filter('TrackID', $trackID) + ->exclude('PageID', 0); + $completedPages = count($result); - public function createQueuedReport() { - if (!Permission::check('ADMIN')) return; + echo json_encode(array( + 'TrackID' => $trackID, + 'Completed' => $completedPages, + 'Total' => $noPages + )); - // setup external links job - $externalLinks = new CheckExternalLinksJob(); - $job = singleton('QueuedJobService'); - $jobID = $job->queueJob($externalLinks); + if ($completedPages >= $noPages) { + $this->clear(); + } else { + $this->continueJob(); + } + } + + /* + * Clears the tracking id and any surplus entries for the BrokenExternalPageTrack model + */ + public function clear() { + // clear any old entries + $trackID = Session::get('ExternalLinksTrackID'); + $oldEntries = BrokenExternalPageTrack::get() + ->exclude('TrackID', $trackID); + foreach ($oldEntries as $entry) { + $entry->delete(); + } + Session::clear('ExternalLinksTrackID'); + } + + /* + * Starts a broken external link check + */ + public function start() { + $track = BrokenExternalPageTrack::create(); + $track->write(); + $track->TrackID = $track->ID; + $track->write(); + + Session::set('ExternalLinksTrackID', $track->ID); + + $this->continueJob(); + } - // redirect to the jobs page - $admin = QueuedJobsAdmin::create(); - $this->Redirect($admin->Link()); + /* + * Continues a broken external link check + */ + public function continueJob() { + $task = new CheckExternalLinks(); + $task->run(null); } } diff --git a/code/jobs/CheckExternalLinksJob.php b/code/jobs/CheckExternalLinksJob.php index ba569ca..af76fc2 100644 --- a/code/jobs/CheckExternalLinksJob.php +++ b/code/jobs/CheckExternalLinksJob.php @@ -56,7 +56,8 @@ public function process() { } $task = new CheckExternalLinks(); - $task->run($page); + $task->pageToProcess = $page; + $task->run(); // and now we store the new list of remaining children $this->pagesToProcess = $remainingPages; diff --git a/code/model/BrokenExternalLink.php b/code/model/BrokenExternalLink.php index 1de3554..89aeeb8 100644 --- a/code/model/BrokenExternalLink.php +++ b/code/model/BrokenExternalLink.php @@ -31,3 +31,13 @@ function canView($member = false) { return Permission::checkMember($member, $codes); } } + +class BrokenExternalPageTrack extends DataObject { + private static $db = array( + 'TrackID' => 'Int' + ); + + private static $has_one = array( + 'Page' => 'Page' + ); +} diff --git a/code/reports/BrokenExternalLinksReport.php b/code/reports/BrokenExternalLinksReport.php index 4bfe568..9e54a46 100644 --- a/code/reports/BrokenExternalLinksReport.php +++ b/code/reports/BrokenExternalLinksReport.php @@ -66,19 +66,20 @@ public function sourceRecords() { } public function getCMSFields() { + Requirements::javascript('externallinks/javascript/BrokenExternalLinksReport.js'); $fields = parent::getCMSFields(); if (class_exists('AbstractQueuedJob')) { - $button = ''; + $button = ''; $runReportButton = new LiteralField( 'runReport', sprintf( $button, - 'admin/externallinks/createQueuedReport', _t('ExternalBrokenLinksReport.RUNREPORT', 'Create new report') ) ); $fields->push($runReportButton); - $reportResultSpan = ''; + + $reportResultSpan = '

'; $reportResult = new LiteralField('ResultTitle', $reportResultSpan); $fields->push($reportResult); } diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index a6ef0b0..b0e2d40 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -1,6 +1,7 @@ completedPages; - } - - public function getTotalPages() { - return $this->totalPages; - } - function run($request) { - if (isset($request->ID)) { - $pages = $request; + $trackID = Session::get('ExternalLinksTrackID'); + if (isset($this->pageToProcess)) { + $pages = $this->pageToProcess; } else { - $pages = Versioned::get_by_stage('SiteTree', 'Live'); + if ($trackID) { + $result = BrokenExternalPageTrack::get() + ->filter('TrackID', $trackID); + $pages = Versioned::get_by_stage('SiteTree', 'Live') + ->exclude('ID', $result->column('PageID')) + ->limit(10); + } else { + $pages = Versioned::get_by_stage('SiteTree', 'Live'); + } } foreach ($pages as $page) { ++$this->totalPages; @@ -93,6 +95,12 @@ function run($request) { } } ++$this->completedPages; + if ($trackID) { + $trackPage = new BrokenExternalPageTrack(); + $trackPage->PageID = $page->ID; + $trackPage->TrackID = $trackID; + $trackPage->write(); + } } // run this again if queued jobs exists and is a valid int diff --git a/javascript/BrokenExternalLinksReport.js b/javascript/BrokenExternalLinksReport.js new file mode 100644 index 0000000..9cf4a32 --- /dev/null +++ b/javascript/BrokenExternalLinksReport.js @@ -0,0 +1,45 @@ +(function($) { + $('#externalLinksReport').entwine({ + onclick: function() { + $(this).start(); + $(this).poll(); + }, + start: function() { + // initiate a new job + $('#ReportHolder').empty(); + $('#ReportHolder').text('Running report 0%'); + $('#ReportHolder').append(''); + $.ajax({url: "admin/externallinks/start", async: true, timeout: 1000 }); + }, + poll: function() { + // poll the current job and update the front end status + $.ajax({ + url: "admin/externallinks/getJobStatus", + async: true, + success: function(data) { + var obj = $.parseJSON(data); + if (!obj) return; + var completed = obj.Completed ? obj.Completed : 0; + var total = obj.Total ? obj.Total : 0; + if (total > 0 && completed == total) { + $('#ReportHolder').text('Report Finished ' + completed + '/' + total); + } else { + setTimeout(function() { $('#externalLinksReport').poll(); }, 1); + } + if (total && completed) { + if (completed < total) { + var percent = (completed / total) * 100; + $('#ReportHolder').text('Running report ' + completed + '/' + + total + ' (' + percent.toFixed(2) + '%)'); + $('#ReportHolder'). + append(''); + } + } + }, + error: function(e) { + console.log(e); + } + }); + } + }); +}(jQuery)); From 564514b01518d6b91c8fb9f4ac926e19b869770f Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Wed, 30 Jul 2014 15:29:24 +1200 Subject: [PATCH 04/11] NEW: Adding whitelisted codes via yml --- README.md | 18 ++++++++++++++---- code/tasks/CheckExternalLinks.php | 11 ++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6999e68..ead445f 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,17 @@ broken links. If you have the queuedjobs module installed you can set the task to be run every so ofter Add the following yml config to config.yml in mysite/_config have the the task run once every day (86400 seconds) -``` -CheckExternalLinks: - Delay: 86400 -``` + CheckExternalLinks: + Delay: 86400 + +## Whitelisting codes ## + +If you want to ignore or whitelist certain http codes this can be setup via IgnoreCodes in the config.yml +file in mysite/_config + + CheckExternalLinks: + Delay: 60 + IgnoreCodes: + - 401 + - 403 + - 501 diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index b0e2d40..e63815b 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -61,9 +61,14 @@ function run($request) { $response = curl_exec($handle); $httpCode = curl_getinfo($handle, CURLINFO_HTTP_CODE); curl_close($handle); - if (($httpCode < 200 || $httpCode > 302) - || ($href == '' || $href[0] == '/')) - { + // do we have any whitelisted codes + $ignoreCodes = Config::inst()->get('CheckExternalLinks', 'IgnoreCodes'); + // if the code is whitelisted set it to 200 + $httpCode = (is_array($ignoreCodes) && in_array($httpCode, $ignoreCodes)) ? + 200 : $httpCode; + + // ignore empty hrefs and internal links + if (($httpCode < 200 || $httpCode > 302) || ($href == '' || $href[0] == '/')) { $brokenLink = new BrokenExternalLink(); $brokenLink->PageID = $page->ID; $brokenLink->Link = $href; From e9fe1a470720b29ebb1436b93a6f43e05a783c94 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Thu, 31 Jul 2014 16:49:20 +1200 Subject: [PATCH 05/11] NEW: Use DB row for job status and refactor the sql statements --- code/controllers/CMSExternalLinks.php | 71 ++++-------- code/jobs/CheckExternalLinksJob.php | 43 +------- code/model/BrokenExternalLink.php | 12 ++- code/reports/BrokenExternalLinksReport.php | 28 ++--- code/tasks/CheckExternalLinks.php | 120 ++++++++++++++++++--- composer.json | 3 + javascript/BrokenExternalLinksReport.js | 15 ++- tests/ExternalLinksTest.php | 18 ++-- tests/ExternalLinksTest.yml | 2 +- 9 files changed, 180 insertions(+), 132 deletions(-) diff --git a/code/controllers/CMSExternalLinks.php b/code/controllers/CMSExternalLinks.php index 5c23815..8e3ea1a 100644 --- a/code/controllers/CMSExternalLinks.php +++ b/code/controllers/CMSExternalLinks.php @@ -2,69 +2,44 @@ class CMSExternalLinks_Controller extends Controller { - private static $allowed_actions = array('getJobStatus', 'clear', 'start'); + private static $allowed_actions = array('getJobStatus', 'start'); /* * Respond to Ajax requests for info on a running job - * also calls continueJob and clear depending on the status of the job * * @return string JSON string detailing status of the job */ public function getJobStatus() { - $trackID = Session::get('ExternalLinksTrackID'); - if (!$trackID) return; - $noPages = Versioned::get_by_stage('SiteTree', 'Live')->count(); - $result = BrokenExternalPageTrack::get() - ->filter('TrackID', $trackID) - ->exclude('PageID', 0); - $completedPages = count($result); - + $track = CheckExternalLinks::getLatestTrack(); + if (!$track || !$track->exists()) return null; echo json_encode(array( - 'TrackID' => $trackID, - 'Completed' => $completedPages, - 'Total' => $noPages + 'TrackID' => $track->ID, + 'Status' => $track->Status, + 'Completed' => $track->CompletedPages, + 'Total' => $track->TotalPages )); - - if ($completedPages >= $noPages) { - $this->clear(); - } else { - $this->continueJob(); - } } - /* - * Clears the tracking id and any surplus entries for the BrokenExternalPageTrack model - */ - public function clear() { - // clear any old entries - $trackID = Session::get('ExternalLinksTrackID'); - $oldEntries = BrokenExternalPageTrack::get() - ->exclude('TrackID', $trackID); - foreach ($oldEntries as $entry) { - $entry->delete(); - } - Session::clear('ExternalLinksTrackID'); - } /* * Starts a broken external link check */ public function start() { - $track = BrokenExternalPageTrack::create(); - $track->write(); - $track->TrackID = $track->ID; - $track->write(); - - Session::set('ExternalLinksTrackID', $track->ID); - - $this->continueJob(); - } - - /* - * Continues a broken external link check - */ - public function continueJob() { - $task = new CheckExternalLinks(); - $task->run(null); + $status = checkExternalLinks::getLatestTrackStatus(); + // return if the a job is already running + if ($status == 'Running') { + return; + } + if (class_exists('QueuedJobService')) { + $checkLinks = new CheckExternalLinksJob(); + singleton('QueuedJobService') + ->queueJob($checkLinks, date('Y-m-d H:i:s', time() + 1)); + } else { + //TODO this hangs as it waits for the connection to be released + // should return back and continue processing + // http://us3.php.net/manual/en/features.connection-handling.php + $task = new CheckExternalLinks(); + $task->run(); + } } } diff --git a/code/jobs/CheckExternalLinksJob.php b/code/jobs/CheckExternalLinksJob.php index af76fc2..030f1ea 100644 --- a/code/jobs/CheckExternalLinksJob.php +++ b/code/jobs/CheckExternalLinksJob.php @@ -8,12 +8,6 @@ */ class CheckExternalLinksJob extends AbstractQueuedJob implements QueuedJob { - public function __construct() { - $this->pagesToProcess = Versioned::get_by_stage('SiteTree', 'Live')->column(); - $this->currentStep = 0; - $this->totalSteps = count($this->pagesToProcess); - } - public function getTitle() { return _t('CheckExternalLiksJob.TITLE', 'Checking for external broken links'); } @@ -26,47 +20,14 @@ public function getSignature() { return md5(get_class($this)); } - public function setup() { - parent::setup(); - $restart = $this->currentStep == 0; - if ($restart) { - $this->pagesToProcess = Versioned::get_by_stage('SiteTree', 'Live')->column(); - } - - } - /** * Check a individual page */ public function process() { - $remainingPages = $this->pagesToProcess; - if (!count($remainingPages)) { - $this->isComplete = true; - return; - } - - // lets process our first item - note that we take it off the list of things left to do - $ID = array_shift($remainingPages); - - // get the page - $page = Versioned::get_by_stage('SiteTree', 'Live', 'ID = '.$ID); - - if (!$page || !$page->Count()) { - $this->addMessage("Page ID #$ID could not be found, skipping"); - } - $task = new CheckExternalLinks(); - $task->pageToProcess = $page; $task->run(); - - // and now we store the new list of remaining children - $this->pagesToProcess = $remainingPages; - $this->currentStep++; - - if (!count($remainingPages)) { - $this->isComplete = true; - return; - } + $this->isComplete = true; + return; } } diff --git a/code/model/BrokenExternalLink.php b/code/model/BrokenExternalLink.php index 89aeeb8..a716aea 100644 --- a/code/model/BrokenExternalLink.php +++ b/code/model/BrokenExternalLink.php @@ -32,9 +32,19 @@ function canView($member = false) { } } +class BrokenExternalPageTrackStatus extends DataObject { + private static $db = array( + 'Status' => 'Enum("Completed, Running", "Running")', + 'TotalPages' => 'Int', + 'CompletedPages' => 'Int', + 'JobInfo' => 'Varchar(255)' + ); +} + class BrokenExternalPageTrack extends DataObject { private static $db = array( - 'TrackID' => 'Int' + 'TrackID' => 'Int', + 'Processed' => 'Boolean' ); private static $has_one = array( diff --git a/code/reports/BrokenExternalLinksReport.php b/code/reports/BrokenExternalLinksReport.php index 9e54a46..a57e976 100644 --- a/code/reports/BrokenExternalLinksReport.php +++ b/code/reports/BrokenExternalLinksReport.php @@ -68,21 +68,21 @@ public function sourceRecords() { public function getCMSFields() { Requirements::javascript('externallinks/javascript/BrokenExternalLinksReport.js'); $fields = parent::getCMSFields(); - if (class_exists('AbstractQueuedJob')) { - $button = ''; - $runReportButton = new LiteralField( - 'runReport', - sprintf( - $button, - _t('ExternalBrokenLinksReport.RUNREPORT', 'Create new report') - ) - ); - $fields->push($runReportButton); - $reportResultSpan = '

'; - $reportResult = new LiteralField('ResultTitle', $reportResultSpan); - $fields->push($reportResult); - } + $reportResultSpan = '

'; + $reportResult = new LiteralField('ResultTitle', $reportResultSpan); + $fields->push($reportResult); + + $button = ''; + $runReportButton = new LiteralField( + 'runReport', + sprintf( + $button, + _t('ExternalBrokenLinksReport.RUNREPORT', 'Create new report') + ) + ); + $fields->push($runReportButton); + return $fields; } } diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index e63815b..960987f 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -12,23 +12,63 @@ class CheckExternalLinks extends BuildTask { private $totalPages; function run($request) { - $trackID = Session::get('ExternalLinksTrackID'); - if (isset($this->pageToProcess)) { - $pages = $this->pageToProcess; + $track = CheckExternalLinks::getLatestTrack(); + + // if the script has already been started + if ($track && $track->Status == 'Running') { + $batch = BrokenExternalPageTrack::get() + ->filter(array( + 'TrackID' => $track->ID, + 'Processed' => 0 + ))->limit(10)->column('PageID'); + $pages = Versioned::get_by_stage('SiteTree', 'Live') + ->filter('ID', $batch) + ->limit(10); + $this->updateJobInfo('Fetching pages to check'); + if ($track->CompletedPages == $track->TotalPages) { + $track->Status = 'Completed'; + $track->write(); + $this->updateJobInfo('Setting to completed'); + } + // if the script is to be started } else { - if ($trackID) { - $result = BrokenExternalPageTrack::get() - ->filter('TrackID', $trackID); - $pages = Versioned::get_by_stage('SiteTree', 'Live') - ->exclude('ID', $result->column('PageID')) - ->limit(10); - } else { - $pages = Versioned::get_by_stage('SiteTree', 'Live'); + $pages = Versioned::get_by_stage('SiteTree', 'Live')->column('ID'); + $noPages = count($pages); + + $track = BrokenExternalPageTrackStatus::create(); + $track->TotalPages = $noPages; + $track->write(); + $this->updateJobInfo('Creating new tracking object'); + + foreach ($pages as $page) { + $trackPage = BrokenExternalPageTrack::create(); + $trackPage->PageID = $page; + $trackPage->TrackID = $track->ID; + $trackPage->write(); } + + $batch = BrokenExternalPageTrack::get() + ->filter(array( + 'TrackID' => $track->ID + ))->limit(10)->column('PageID'); + + $pages = Versioned::get_by_stage('SiteTree', 'Live') + ->filter('ID', $batch); } + $trackID = $track->ID; foreach ($pages as $page) { ++$this->totalPages; + if ($track->ID) { + $trackPage = BrokenExternalPageTrack::get() + ->filter(array( + 'PageID' => $page->ID, + 'TrackID' => $track->ID + ))->first(); + $trackPage->Processed = 1; + $trackPage->write(); + } + $htmlValue = Injector::inst()->create('HTMLValue', $page->Content); if (!$htmlValue->isValid()) { continue; @@ -100,12 +140,27 @@ function run($request) { } } ++$this->completedPages; - if ($trackID) { - $trackPage = new BrokenExternalPageTrack(); - $trackPage->PageID = $page->ID; - $trackPage->TrackID = $trackID; - $trackPage->write(); + } + + // run this outside the foreach loop to stop it locking DB rows + $this->updateJobInfo('Updating completed pages'); + $this->updateCompletedPages($trackID); + + // do we need to carry on running the job + $track = $this->getLatestTrack(); + if ($track->CompletedPages >= $track->TotalPages) { + $track->Status = 'Completed'; + $track->write(); + + // clear any old previous data + $rows = BrokenExternalPageTrack::get() + ->exclude('TrackID', $track->ID); + foreach ($rows as $row) { + $row->delete(); } + } else { + $this->updateJobInfo("Running next batch {$track->CompletedPages}/{$track->TotalPages}"); + $this->run($request); } // run this again if queued jobs exists and is a valid int @@ -115,6 +170,39 @@ function run($request) { singleton('QueuedJobService') ->queueJob($checkLinks, date('Y-m-d H:i:s', time() + $queuedJob)); } + } + + public static function getLatestTrack() { + $track = BrokenExternalPageTrackStatus::get()->sort('ID', 'DESC')->first(); + if (!$track || !$track->exists()) return null; + return $track; + } + + public static function getLatestTrackID() { + $track = CheckExternalLinks::getLatestTrack(); + if (!$track || !$track->exists()) return null; + return $track->ID; + } + + public static function getLatestTrackStatus() { + $track = CheckExternalLinks::getLatestTrack(); + if (!$track || !$track->exists()) return null; + return $track->Status; + } + + private function updateCompletedPages($trackID = 0) { + $noPages = BrokenExternalPageTrack::get() + ->filter(array('TrackID' => $trackID, 'Processed' => 1))->count(); + $track = $this->getLatestTrack($trackID); + $track->CompletedPages = $noPages; + $track->write(); + return $noPages; + } + private function updateJobInfo($message) { + $track = CheckExternalLinks::getLatestTrack(); + if (!$track || !$track->exists()) return null; + $track->JobInfo = $message; + $track->write(); } } diff --git a/composer.json b/composer.json index 7f73048..36506f7 100644 --- a/composer.json +++ b/composer.json @@ -13,5 +13,8 @@ { "silverstripe/framework": ">=3.0", "silverstripe/cms": ">=3.0" + }, + "suggest": { + "silverstripe/queuedjobs": "Speeds up running the job for Content Editors fropm the report" } } diff --git a/javascript/BrokenExternalLinksReport.js b/javascript/BrokenExternalLinksReport.js index 9cf4a32..5dd7046 100644 --- a/javascript/BrokenExternalLinksReport.js +++ b/javascript/BrokenExternalLinksReport.js @@ -4,12 +4,16 @@ $(this).start(); $(this).poll(); }, + onmatch: function() { + $(this).poll(); + }, start: function() { // initiate a new job $('#ReportHolder').empty(); $('#ReportHolder').text('Running report 0%'); $('#ReportHolder').append(''); - $.ajax({url: "admin/externallinks/start", async: true, timeout: 1000 }); + $.ajax({url: "admin/externallinks/start", async: true, timeout: 3000 }); + $(this).poll(); }, poll: function() { // poll the current job and update the front end status @@ -18,13 +22,16 @@ async: true, success: function(data) { var obj = $.parseJSON(data); - if (!obj) return; + if (!obj) { + setTimeout(function() { $('#externalLinksReport').poll(); }, 1000); + } var completed = obj.Completed ? obj.Completed : 0; var total = obj.Total ? obj.Total : 0; - if (total > 0 && completed == total) { + var jobStatus = obj.Status ? obj.Status : 'Running'; + if (jobStatus == 'Completed') { $('#ReportHolder').text('Report Finished ' + completed + '/' + total); } else { - setTimeout(function() { $('#externalLinksReport').poll(); }, 1); + setTimeout(function() { $('#externalLinksReport').poll(); }, 1000); } if (total && completed) { if (completed < total) { diff --git a/tests/ExternalLinksTest.php b/tests/ExternalLinksTest.php index ae9d898..a9b813f 100644 --- a/tests/ExternalLinksTest.php +++ b/tests/ExternalLinksTest.php @@ -4,25 +4,28 @@ class ExternalLinks extends FunctionalTest { protected static $fixture_file = 'ExternalLinksTest.yml'; - public function testWorkingLink() { + public function testLinks() { // uses http://127.0.0.1 to test a working link - $page = $this->objFromFixture('Page', 'working'); + $working = $this->objFromFixture('SiteTree', 'working'); + $working->publish('Stage', 'Live'); $task = new CheckExternalLinks(); - $task->run($page); - $brokenLinks = BrokenExternalLinks::get(); + $task->run(null); + $brokenLinks = BrokenExternalLink::get(); $this->assertEquals(0, $brokenLinks->count()); } public function testBrokenLink() { // uses http://192.0.2.1 for a broken link - $page = $this->objFromFixture('Page', 'broken'); + $broken = $this->objFromFixture('SiteTree', 'broken'); + $broken->publish('Stage', 'Live'); $task = new CheckExternalLinks(); - $task->run($page); - $brokenLinks = BrokenExternalLinks::get(); + $task->run(null); + $brokenLinks = BrokenExternalLink::get(); $this->assertEquals(1, $brokenLinks->count()); } public function testReportExists() { + $mock = $this->objFromFixture('SiteTree', 'broken'); $reports = SS_Report::get_reports(); $reportNames = array(); foreach($reports as $report) { @@ -32,3 +35,4 @@ public function testReportExists() { 'BrokenExternalLinksReport is in reports list'); } } + diff --git a/tests/ExternalLinksTest.yml b/tests/ExternalLinksTest.yml index 8a25106..46eb0a8 100644 --- a/tests/ExternalLinksTest.yml +++ b/tests/ExternalLinksTest.yml @@ -1,4 +1,4 @@ -Page: +SiteTree: working: Title: Working Link Content: 'Localhost' From 9e5a41f6c7f2c27062c8afe1788797ac0dc4b9f2 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Mon, 4 Aug 2014 10:10:59 +1200 Subject: [PATCH 06/11] NEW: Fixing queuedjob status and hiding report button when report is running --- code/jobs/CheckExternalLinksJob.php | 9 ++++++++- code/tasks/CheckExternalLinks.php | 18 ++++++++++++------ javascript/BrokenExternalLinksReport.js | 4 +++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/code/jobs/CheckExternalLinksJob.php b/code/jobs/CheckExternalLinksJob.php index 030f1ea..56dd1e3 100644 --- a/code/jobs/CheckExternalLinksJob.php +++ b/code/jobs/CheckExternalLinksJob.php @@ -25,7 +25,14 @@ public function getSignature() { */ public function process() { $task = new CheckExternalLinks(); - $task->run(); + $pages = Versioned::get_by_stage('SiteTree', 'Live'); + // set the limit so each page is done individually + $task->limit = 1; + $this->totalSteps = $pages->count(); + foreach ($pages as $page) { + $this->currentStep++; + $task->run(); + } $this->isComplete = true; return; } diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index 960987f..f3df02c 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -1,7 +1,8 @@ filter(array( 'TrackID' => $track->ID, 'Processed' => 0 - ))->limit(10)->column('PageID'); + ))->limit($this->limit)->column('PageID'); $pages = Versioned::get_by_stage('SiteTree', 'Live') ->filter('ID', $batch) - ->limit(10); + ->limit($this->limit); $this->updateJobInfo('Fetching pages to check'); if ($track->CompletedPages == $track->TotalPages) { $track->Status = 'Completed'; @@ -50,7 +51,7 @@ function run($request) { $batch = BrokenExternalPageTrack::get() ->filter(array( 'TrackID' => $track->ID - ))->limit(10)->column('PageID'); + ))->limit($this->limit)->column('PageID'); $pages = Versioned::get_by_stage('SiteTree', 'Live') ->filter('ID', $batch); @@ -159,8 +160,13 @@ function run($request) { $row->delete(); } } else { - $this->updateJobInfo("Running next batch {$track->CompletedPages}/{$track->TotalPages}"); - $this->run($request); + // if running via the queued job module return to the queued job after each iteration + if ($this->limit == 1) { + return; + } else { + $this->updateJobInfo("Running next batch {$track->CompletedPages}/{$track->TotalPages}"); + $this->run($request); + } } // run this again if queued jobs exists and is a valid int diff --git a/javascript/BrokenExternalLinksReport.js b/javascript/BrokenExternalLinksReport.js index 5dd7046..54174f4 100644 --- a/javascript/BrokenExternalLinksReport.js +++ b/javascript/BrokenExternalLinksReport.js @@ -2,7 +2,6 @@ $('#externalLinksReport').entwine({ onclick: function() { $(this).start(); - $(this).poll(); }, onmatch: function() { $(this).poll(); @@ -12,11 +11,13 @@ $('#ReportHolder').empty(); $('#ReportHolder').text('Running report 0%'); $('#ReportHolder').append(''); + $('#externalLinksReport').hide(); $.ajax({url: "admin/externallinks/start", async: true, timeout: 3000 }); $(this).poll(); }, poll: function() { // poll the current job and update the front end status + $('#externalLinksReport').hide(); $.ajax({ url: "admin/externallinks/getJobStatus", async: true, @@ -30,6 +31,7 @@ var jobStatus = obj.Status ? obj.Status : 'Running'; if (jobStatus == 'Completed') { $('#ReportHolder').text('Report Finished ' + completed + '/' + total); + $('#externalLinksReport').show(); } else { setTimeout(function() { $('#externalLinksReport').poll(); }, 1000); } From 363ecd49856c26520085a7442a2478906e46336a Mon Sep 17 00:00:00 2001 From: Damian Mooyman Date: Mon, 4 Aug 2014 11:20:54 +1200 Subject: [PATCH 07/11] Make sure that process yields to queuedjobs after each page Fix javascript errors starting the job --- code/jobs/CheckExternalLinksJob.php | 16 ++++------- code/tasks/CheckExternalLinks.php | 35 ++++++++++++++----------- javascript/BrokenExternalLinksReport.js | 11 +++++--- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/code/jobs/CheckExternalLinksJob.php b/code/jobs/CheckExternalLinksJob.php index 56dd1e3..bb79e67 100644 --- a/code/jobs/CheckExternalLinksJob.php +++ b/code/jobs/CheckExternalLinksJob.php @@ -21,20 +21,14 @@ public function getSignature() { } /** - * Check a individual page + * Check an individual page */ public function process() { $task = new CheckExternalLinks(); - $pages = Versioned::get_by_stage('SiteTree', 'Live'); - // set the limit so each page is done individually - $task->limit = 1; - $this->totalSteps = $pages->count(); - foreach ($pages as $page) { - $this->currentStep++; - $task->run(); - } - $this->isComplete = true; - return; + $track = $task->runLinksCheck(1); + $this->currentStep = $track->CompletedPages; + $this->totalSteps = $track->TotalPages; + $this->isComplete = $track->Status === 'Completed'; } } diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index f3df02c..e9e0cc0 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -13,6 +13,16 @@ class CheckExternalLinks extends BuildTask { private $totalPages; function run($request) { + $this->runLinksCheck($this->limit); + } + + /** + * Runs the links checker and returns the track used + * + * @param int $limit Limit to number of pages to run + * @return BrokenExternalPageTrackStatus + */ + public function runLinksCheck($limit) { $track = CheckExternalLinks::getLatestTrack(); // if the script has already been started @@ -21,10 +31,10 @@ function run($request) { ->filter(array( 'TrackID' => $track->ID, 'Processed' => 0 - ))->limit($this->limit)->column('PageID'); + ))->limit($limit)->column('PageID'); $pages = Versioned::get_by_stage('SiteTree', 'Live') ->filter('ID', $batch) - ->limit($this->limit); + ->limit($limit); $this->updateJobInfo('Fetching pages to check'); if ($track->CompletedPages == $track->TotalPages) { $track->Status = 'Completed'; @@ -51,7 +61,7 @@ function run($request) { $batch = BrokenExternalPageTrack::get() ->filter(array( 'TrackID' => $track->ID - ))->limit($this->limit)->column('PageID'); + ))->limit($limit)->column('PageID'); $pages = Versioned::get_by_stage('SiteTree', 'Live') ->filter('ID', $batch); @@ -159,22 +169,15 @@ function run($request) { foreach ($rows as $row) { $row->delete(); } - } else { + return $track; + } + // if running via the queued job module return to the queued job after each iteration - if ($this->limit == 1) { - return; + if ($limit == 1) { + return $track; } else { $this->updateJobInfo("Running next batch {$track->CompletedPages}/{$track->TotalPages}"); - $this->run($request); - } - } - - // run this again if queued jobs exists and is a valid int - $queuedJob = Config::inst()->get('CheckExternalLinks', 'Delay'); - if (isset($queuedJob) && is_int($queuedJob) && class_exists('QueuedJobService')) { - $checkLinks = new CheckExternalLinksJob(); - singleton('QueuedJobService') - ->queueJob($checkLinks, date('Y-m-d H:i:s', time() + $queuedJob)); + return $this->runLinksCheck($limit); } } diff --git a/javascript/BrokenExternalLinksReport.js b/javascript/BrokenExternalLinksReport.js index 54174f4..d44f7b5 100644 --- a/javascript/BrokenExternalLinksReport.js +++ b/javascript/BrokenExternalLinksReport.js @@ -4,6 +4,8 @@ $(this).start(); }, onmatch: function() { + // poll the current job and update the front end status + $('#externalLinksReport').hide(); $(this).poll(); }, start: function() { @@ -16,15 +18,16 @@ $(this).poll(); }, poll: function() { - // poll the current job and update the front end status - $('#externalLinksReport').hide(); $.ajax({ url: "admin/externallinks/getJobStatus", async: true, success: function(data) { var obj = $.parseJSON(data); + + // No report, so let user create one if (!obj) { - setTimeout(function() { $('#externalLinksReport').poll(); }, 1000); + $('#externalLinksReport').show(); + return; } var completed = obj.Completed ? obj.Completed : 0; var total = obj.Total ? obj.Total : 0; @@ -46,7 +49,7 @@ } }, error: function(e) { - console.log(e); + if(typeof console !== 'undefined') console.log(e); } }); } From da7e45ddd41161d90d92948573293a3f0fc03fe2 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Mon, 4 Aug 2014 12:30:08 +1200 Subject: [PATCH 08/11] BUG: Fixing report page link and set to check stage not live --- code/reports/BrokenExternalLinksReport.php | 1 + code/tasks/CheckExternalLinks.php | 8 ++++---- tests/ExternalLinksTest.php | 9 +++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/code/reports/BrokenExternalLinksReport.php b/code/reports/BrokenExternalLinksReport.php index a57e976..34bfbec 100644 --- a/code/reports/BrokenExternalLinksReport.php +++ b/code/reports/BrokenExternalLinksReport.php @@ -60,6 +60,7 @@ public function sourceRecords() { $links = BrokenExternalLink::get(); foreach ($links as $link) { $link->PageLink = $link->Page()->Title; + $link->ID = $link->Page()->ID; $returnSet->push($link); } return $returnSet; diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index e9e0cc0..7d4a605 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -32,7 +32,7 @@ public function runLinksCheck($limit) { 'TrackID' => $track->ID, 'Processed' => 0 ))->limit($limit)->column('PageID'); - $pages = Versioned::get_by_stage('SiteTree', 'Live') + $pages = Versioned::get_by_stage('SiteTree', 'Stage') ->filter('ID', $batch) ->limit($limit); $this->updateJobInfo('Fetching pages to check'); @@ -43,7 +43,7 @@ public function runLinksCheck($limit) { } // if the script is to be started } else { - $pages = Versioned::get_by_stage('SiteTree', 'Live')->column('ID'); + $pages = Versioned::get_by_stage('SiteTree', 'Stage')->column('ID'); $noPages = count($pages); $track = BrokenExternalPageTrackStatus::create(); @@ -63,7 +63,7 @@ public function runLinksCheck($limit) { 'TrackID' => $track->ID ))->limit($limit)->column('PageID'); - $pages = Versioned::get_by_stage('SiteTree', 'Live') + $pages = Versioned::get_by_stage('SiteTree', 'Stage') ->filter('ID', $batch); } $trackID = $track->ID; @@ -139,7 +139,7 @@ public function runLinksCheck($limit) { // bypass the ORM as syncLinkTracking does not allow you // to update HasBrokenLink to true - $query = "UPDATE \"SiteTree_Live\" SET \"HasBrokenLink\" = 1 "; + $query = "UPDATE \"SiteTree\" SET \"HasBrokenLink\" = 1 "; $query .= "WHERE \"ID\" = " . (int)$page->ID; $result = DB::query($query); if (!$result) { diff --git a/tests/ExternalLinksTest.php b/tests/ExternalLinksTest.php index a9b813f..181365a 100644 --- a/tests/ExternalLinksTest.php +++ b/tests/ExternalLinksTest.php @@ -7,17 +7,18 @@ class ExternalLinks extends FunctionalTest { public function testLinks() { // uses http://127.0.0.1 to test a working link $working = $this->objFromFixture('SiteTree', 'working'); - $working->publish('Stage', 'Live'); + $working->publish('Stage', 'Stage'); $task = new CheckExternalLinks(); $task->run(null); - $brokenLinks = BrokenExternalLink::get(); - $this->assertEquals(0, $brokenLinks->count()); + $brokenLinks = BrokenExternalLink::get()->column('Link');; + // confirm the working link has not been added as a broken link + $this->assertNotEquals($working->Link, $brokenLinks[0]); } public function testBrokenLink() { // uses http://192.0.2.1 for a broken link $broken = $this->objFromFixture('SiteTree', 'broken'); - $broken->publish('Stage', 'Live'); + $broken->publish('Stage', 'Stage'); $task = new CheckExternalLinks(); $task->run(null); $brokenLinks = BrokenExternalLink::get(); From 6bda4dcc82fbb6f4df56aa430dcec93c1435a609 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Mon, 4 Aug 2014 17:38:28 +1200 Subject: [PATCH 09/11] BUG: Correcting test code and fixing bug with create report button --- code/controllers/CMSExternalLinks.php | 15 +++++++++++++++ javascript/BrokenExternalLinksReport.js | 10 +++++----- tests/ExternalLinksTest.php | 4 ++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/code/controllers/CMSExternalLinks.php b/code/controllers/CMSExternalLinks.php index 8e3ea1a..8579bdd 100644 --- a/code/controllers/CMSExternalLinks.php +++ b/code/controllers/CMSExternalLinks.php @@ -31,6 +31,21 @@ public function start() { return; } if (class_exists('QueuedJobService')) { + $pages = Versioned::get_by_stage('SiteTree', 'Stage'); + $noPages = count($pages); + + $track = BrokenExternalPageTrackStatus::create(); + $track->TotalPages = $noPages; + $track->Status = 'Running'; + $track->write(); + + foreach ($pages as $page) { + $trackPage = BrokenExternalPageTrack::create(); + $trackPage->PageID = $page->ID; + $trackPage->TrackID = $track->ID; + $trackPage->write(); + } + $checkLinks = new CheckExternalLinksJob(); singleton('QueuedJobService') ->queueJob($checkLinks, date('Y-m-d H:i:s', time() + 1)); diff --git a/javascript/BrokenExternalLinksReport.js b/javascript/BrokenExternalLinksReport.js index d44f7b5..ff9acfb 100644 --- a/javascript/BrokenExternalLinksReport.js +++ b/javascript/BrokenExternalLinksReport.js @@ -6,7 +6,7 @@ onmatch: function() { // poll the current job and update the front end status $('#externalLinksReport').hide(); - $(this).poll(); + $(this).poll(0); }, start: function() { // initiate a new job @@ -15,9 +15,9 @@ $('#ReportHolder').append(''); $('#externalLinksReport').hide(); $.ajax({url: "admin/externallinks/start", async: true, timeout: 3000 }); - $(this).poll(); + $(this).poll(1); }, - poll: function() { + poll: function(start) { $.ajax({ url: "admin/externallinks/getJobStatus", async: true, @@ -32,11 +32,11 @@ var completed = obj.Completed ? obj.Completed : 0; var total = obj.Total ? obj.Total : 0; var jobStatus = obj.Status ? obj.Status : 'Running'; - if (jobStatus == 'Completed') { + if (jobStatus == 'Completed' && start == 0) { $('#ReportHolder').text('Report Finished ' + completed + '/' + total); $('#externalLinksReport').show(); } else { - setTimeout(function() { $('#externalLinksReport').poll(); }, 1000); + setTimeout(function() { $('#externalLinksReport').poll(0); }, 1000); } if (total && completed) { if (completed < total) { diff --git a/tests/ExternalLinksTest.php b/tests/ExternalLinksTest.php index 181365a..6c7bf61 100644 --- a/tests/ExternalLinksTest.php +++ b/tests/ExternalLinksTest.php @@ -7,7 +7,7 @@ class ExternalLinks extends FunctionalTest { public function testLinks() { // uses http://127.0.0.1 to test a working link $working = $this->objFromFixture('SiteTree', 'working'); - $working->publish('Stage', 'Stage'); + $working->write(); $task = new CheckExternalLinks(); $task->run(null); $brokenLinks = BrokenExternalLink::get()->column('Link');; @@ -18,7 +18,7 @@ public function testLinks() { public function testBrokenLink() { // uses http://192.0.2.1 for a broken link $broken = $this->objFromFixture('SiteTree', 'broken'); - $broken->publish('Stage', 'Stage'); + $broken->write(); $task = new CheckExternalLinks(); $task->run(null); $brokenLinks = BrokenExternalLink::get(); From f55a650d244f27f8f9caf733ec82bc1d18395a89 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Wed, 6 Aug 2014 15:11:23 +1200 Subject: [PATCH 10/11] NEW: Plat-59 Do not display broken previous report info for the latest report --- code/model/BrokenExternalLink.php | 3 ++- code/reports/BrokenExternalLinksReport.php | 11 +++++++++-- code/tasks/CheckExternalLinks.php | 1 + 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/code/model/BrokenExternalLink.php b/code/model/BrokenExternalLink.php index a716aea..24ff85f 100644 --- a/code/model/BrokenExternalLink.php +++ b/code/model/BrokenExternalLink.php @@ -8,7 +8,8 @@ class BrokenExternalLink extends DataObject { ); private static $has_one = array( - 'Page' => 'Page' + 'Page' => 'Page', + 'Track' => 'BrokenExternalLink' ); public static $summary_fields = array( diff --git a/code/reports/BrokenExternalLinksReport.php b/code/reports/BrokenExternalLinksReport.php index 34bfbec..c908820 100644 --- a/code/reports/BrokenExternalLinksReport.php +++ b/code/reports/BrokenExternalLinksReport.php @@ -34,7 +34,8 @@ public function init() { * @return string */ public function title() { - return _t('ExternalBrokenLinksReport.EXTERNALBROKENLINKS',"External broken links report"); + return _t('ExternalBrokenLinksReport.EXTERNALBROKENLINKS', + "External broken links report"); } /** @@ -56,8 +57,14 @@ public function getColumns() { } public function sourceRecords() { + $track = CheckExternalLinks::getLatestTrack(); $returnSet = new ArrayList(); - $links = BrokenExternalLink::get(); + if ($track && $track->exists()) { + $links = BrokenExternalLink::get() + ->filter('TrackID', $track->ID); + } else { + $links = BrokenExternalLink::get(); + } foreach ($links as $link) { $link->PageLink = $link->Page()->Title; $link->ID = $link->Page()->ID; diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index 7d4a605..4fa8dce 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -124,6 +124,7 @@ public function runLinksCheck($limit) { $brokenLink->PageID = $page->ID; $brokenLink->Link = $href; $brokenLink->HTTPCode = $httpCode; + $brokenLink->TrackID = $track->ID; $brokenLink->write(); // set the broken link class From 96bab169ed1c913e1f815820b7dadbcc8adce998 Mon Sep 17 00:00:00 2001 From: Kirk Mayo Date: Tue, 12 Aug 2014 10:03:41 +1200 Subject: [PATCH 11/11] Revert "NEW: Plat-59 Do not display broken previous report info for the latest report" This reverts commit f55a650d244f27f8f9caf733ec82bc1d18395a89. --- code/model/BrokenExternalLink.php | 3 +-- code/reports/BrokenExternalLinksReport.php | 11 ++--------- code/tasks/CheckExternalLinks.php | 1 - 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/code/model/BrokenExternalLink.php b/code/model/BrokenExternalLink.php index 24ff85f..a716aea 100644 --- a/code/model/BrokenExternalLink.php +++ b/code/model/BrokenExternalLink.php @@ -8,8 +8,7 @@ class BrokenExternalLink extends DataObject { ); private static $has_one = array( - 'Page' => 'Page', - 'Track' => 'BrokenExternalLink' + 'Page' => 'Page' ); public static $summary_fields = array( diff --git a/code/reports/BrokenExternalLinksReport.php b/code/reports/BrokenExternalLinksReport.php index c908820..34bfbec 100644 --- a/code/reports/BrokenExternalLinksReport.php +++ b/code/reports/BrokenExternalLinksReport.php @@ -34,8 +34,7 @@ public function init() { * @return string */ public function title() { - return _t('ExternalBrokenLinksReport.EXTERNALBROKENLINKS', - "External broken links report"); + return _t('ExternalBrokenLinksReport.EXTERNALBROKENLINKS',"External broken links report"); } /** @@ -57,14 +56,8 @@ public function getColumns() { } public function sourceRecords() { - $track = CheckExternalLinks::getLatestTrack(); $returnSet = new ArrayList(); - if ($track && $track->exists()) { - $links = BrokenExternalLink::get() - ->filter('TrackID', $track->ID); - } else { - $links = BrokenExternalLink::get(); - } + $links = BrokenExternalLink::get(); foreach ($links as $link) { $link->PageLink = $link->Page()->Title; $link->ID = $link->Page()->ID; diff --git a/code/tasks/CheckExternalLinks.php b/code/tasks/CheckExternalLinks.php index 4fa8dce..7d4a605 100644 --- a/code/tasks/CheckExternalLinks.php +++ b/code/tasks/CheckExternalLinks.php @@ -124,7 +124,6 @@ public function runLinksCheck($limit) { $brokenLink->PageID = $page->ID; $brokenLink->Link = $href; $brokenLink->HTTPCode = $httpCode; - $brokenLink->TrackID = $track->ID; $brokenLink->write(); // set the broken link class