NEW Move code from silverstripe/externallinks

.stylelintrc.js

@@ -1 +1 @@
-module.exports = require('@silverstripe/eslint-config/.stylelintrc');
+module.exports = require('@silverstripe/eslint-config/.stylelintrc');

README.md

@@ -17,6 +17,7 @@ Silverstripe backend.
 There are also a few CMS reports that comes out of the box:
 - A "Users, Groups and Permissions" report allowing administrators to get a quick overview of who has access to the CMS.
 - A "Site-wide content report" report allowing CMS users to get a quick overview of content across the site.
+- An "External broken links report" allowing users with permissions to track broken external links.
 
 ## Troubleshooting
 
@@ -35,7 +36,9 @@ SilverStripe\Reports\Report:
 ```
 Note that some reports may have overridden the `getCount` method, and for those reports this may not apply.
 
-## Customising the "Site-wide content report"
+## Site-wide content report
+
+### Customising the report
 
 In order to customise the columns included in a report you can build a custom extension and apply it to the
 SitewideContentReport as necessary.
@@ -78,9 +81,9 @@ the following key => value pairs:
  * `casting`: Specify a field type (e.g. `Text` or `Int`) in order to assist with field casting. This is not
     necessary if `formatting` is used.
 
-## Performance considerations
+### Performance considerations
 
-### Large data sets
+#### Large data sets
 
 If your project has a large number of pages or files, you may experience server timeouts while trying to export
 this report to CSV. To avoid this issue, you can either increase your server timeout limit, or you can install
@@ -114,6 +117,102 @@ SilverStripe\Reports\SitewideContentReport\SitewideContentReport:
     - SitewideContentReportQueuedExportExtension
 ```
 
+## External broken links report
+
+### Features
+
+* Add external links to broken links reports
+* Add a task to track external broken links
+
+### Report
+
+A new report is added called 'External Broken links report'. When viewing this report, a user may press
+the "Create new report" button which will trigger an ajax request to initiate a report run.
+
+In this initial ajax request this module will do one of two things, depending on which modules are included:
+
+* If the queuedjobs module is installed, a new queued job will be initiated. The queuedjobs module will then
+  manage the progress of the task.
+* If the queuedjobs module is absent, then the controller will fallback to running a buildtask in the background.
+  This is less robust, as a failure or error during this process will abort the run.
+
+In either case, the background task will loop over every page in the system, inspecting all external urls and
+checking the status code returned by requesting each one. If a URL returns a response code that is considered
+"broken" (defined as < 200 or > 302) then the `ss-broken` css class will be assigned to that url, and
+a line item will be added to the report. If a previously broken link has been corrected or fixed, then
+this class is removed.
+
+In the actual report generated the user can click on any broken link item to either view the link in their browser,
+or edit the containing page in the CMS.
+
+While a report is running the current status of this report will be displayed on the report details page, along
+with the status. The user may leave this page and return to it later to view the ongoing status of this report.
+
+Any subsequent report may not be generated until a prior report has completed.
+
+### Dev task
+
+Run `sake tasks:CheckExternalLinksTask` to check your site for external
+broken links.
+
+### Queued job
+
+If you have the queuedjobs module installed you can set the task to be run every so often.
+
+### Whitelisting codes
+
+If you want to ignore or whitelist certain HTTP codes this can be setup via `ignore_codes` in the config.yml
+file in `mysite/_config`:
+
+```yml
+SilverStripe\Reports\ExternalLinks\Tasks\CheckExternalLinksTask:
+  ignore_codes:
+    - 401
+    - 403
+    - 501
+```
+
+### Follow 301 redirects
+
+You may want to follow a redirected URL a example of this would be redirecting from http to https
+can give you a false poitive as the http code of 301 will be returned which will be classed
+as a working link.
+
+To allow redirects to be followed setup the following config in your config.yml
+
+```yaml
+# Follow 301 redirects
+SilverStripe\Reports\ExternalLinks\Tasks\CurlLinkChecker:
+  follow_location: 1
+```
+
+### Bypass cache
+
+By default the task will attempt to cache any results the cache can be bypassed with the
+following config in config.yml.
+
+```yaml
+# Bypass SS_Cache
+SilverStripe\Reports\ExternalLinks\Tasks\CurlLinkChecker::
+  bypass_cache: 1
+```
+
+### Headers
+
+You may want to set headers to be sent with the CURL request (eg: user-agent) to avoid website rejecting the request thinking it is a bot.
+You can set them with the following config in config.yml.
+
+```yaml
+# Headers
+SilverStripe\Reports\ExternalLinks\Tasks\CurlLinkChecker:
+  headers:
+    - 'user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0'
+    - 'accept-encoding: gzip, deflate, br'
+    - 'referer: https://www.domain.com/'
+    - 'sec-fetch-mode: navigate'
+    ...
+```
+
 ## Links ##
 
  * [License](./LICENSE)

_config/externallinks.yml

@@ -0,0 +1,9 @@
+---
+Name: reports-externallinksdependencies
+---
+SilverStripe\Core\Injector\Injector:
+  SilverStripe\Reports\ExternalLinks\Tasks\LinkChecker: SilverStripe\Reports\ExternalLinks\Tasks\CurlLinkChecker
+  Psr\SimpleCache\CacheInterface.CurlLinkChecker:
+    factory: SilverStripe\Core\Cache\CacheFactory
+    constructor:
+      namespace: 'curllinkchecker'

client/dist/styles/BrokenExternalLinksReport.css

@@ -0,0 +1 @@
+.external-links-report__create-report,.external-links-report__report-progress{margin-top:20px}

client/src/js/BrokenExternalLinksReport.js

@@ -0,0 +1,148 @@
+/* global jQuery */
+(function ($) {
+  // eslint-disable-next-line no-shadow
+  $.entwine('ss', ($) => {
+    $('.external-links-report__create-report').entwine({
+      PollTimeout: null,
+      ButtonIsLoading: false,
+      ReloadContent: false,
+
+      onclick(e) {
+        e.preventDefault();
+
+        this.buttonLoading();
+        this.start();
+      },
+
+      onmatch() {
+        // poll the current job and update the front end status
+        this.poll();
+      },
+
+      start() {
+        const self = this;
+        // initiate a new job
+        $('.external-links-report__report-progress')
+          .empty()
+          .text('Running report 0%');
+
+        $.ajax({
+          url: 'admin/externallinks/start',
+          async: true,
+          timeout: 3000,
+          success() {
+            self.setReloadContent(true);
+            self.poll();
+          },
+          error() {
+            self.buttonReset();
+          }
+        });
+      },
+
+      /**
+       * Get the "create report" button selector
+       *
+       * @return {Object}
+       */
+      getButton() {
+        return $('.external-links-report__create-report');
+      },
+
+      /**
+       * Sets the button into a loading state. See LeftAndMain.js.
+       */
+      buttonLoading() {
+        if (this.getButtonIsLoading()) {
+          return;
+        }
+        this.setButtonIsLoading(true);
+
+        const $button = this.getButton();
+
+        // set button to "submitting" state
+        $button.addClass('btn--loading loading');
+        $button.attr('disabled', true);
+
+        if ($button.is('button')) {
+          $button.append($(
+            '<div class="btn__loading-icon">' +
+              '<span class="btn__circle btn__circle--1" />' +
+              '<span class="btn__circle btn__circle--2" />' +
+              '<span class="btn__circle btn__circle--3" />' +
+            '</div>'));
+
+          $button.css(`${$button.outerWidth()}px`);
+        }
+      },
+
+      /**
+       * Reset the button back to its original state after loading. See LeftAndMain.js.
+       */
+      buttonReset() {
+        this.setButtonIsLoading(false);
+
+        const $button = this.getButton();
+
+        $button.removeClass('btn--loading loading');
+        $button.attr('disabled', false);
+        $button.find('.btn__loading-icon').remove();
+        $button.css('width', 'auto');
+      },
+
+      poll() {
+        const self = this;
+        this.buttonLoading();
+
+        $.ajax({
+          url: 'admin/externallinks/getJobStatus',
+          async: true,
+          success(data) {
+            // No report, so let user create one
+            if (!data || (typeof data === 'object' && data.length < 1)) {
+              self.buttonReset();
+              return;
+            }
+
+            // Parse data
+            const completed = data.Completed ? data.Completed : 0;
+            const total = data.Total ? data.Total : 0;
+
+            // If complete status
+            if (data.Status === 'Completed') {
+              if (self.getReloadContent()) {
+                $('.cms-container').loadPanel(document.location.href, null, {}, true, false);
+                self.setReloadContent(false);
+              }
+              $('.external-links-report__report-progress')
+                .text(`Report finished ${completed}/${total}`);
+
+              self.buttonReset();
+              return;
+            }
+
+            // If incomplete update status
+            if (completed < total) {
+              const percent = (completed / total) * 100;
+              $('.external-links-report__report-progress')
+                .text(`Running report  ${completed}/${total} (${percent.toFixed(2)}%)`);
+            }
+
+            // Ensure the regular poll method is run
+            // kill any existing timeout
+            if (self.getPollTimeout() !== null) {
+              clearTimeout(self.getPollTimeout());
+            }
+
+            self.setPollTimeout(setTimeout(() => {
+              $('.external-links-report__create-report').poll();
+            }, 1000));
+          },
+          error() {
+            self.buttonReset();
+          }
+        });
+      }
+    });
+  });
+}(jQuery));

client/src/styles/BrokenExternalLinksReport.scss

@@ -0,0 +1,4 @@
+.external-links-report__create-report,
+.external-links-report__report-progress {
+  margin-top: 20px;
+}

code/ExternalLinks/Controllers/CMSExternalLinksController.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace SilverStripe\Reports\ExternalLinks\Controllers;
+
+use SilverStripe\Admin\AdminController;
+use SilverStripe\Control\HTTPResponse;
+use SilverStripe\Reports\ExternalLinks\Model\BrokenExternalPageTrackStatus;
+use SilverStripe\Reports\ExternalLinks\Jobs\CheckExternalLinksJob;
+use SilverStripe\Reports\ExternalLinks\Tasks\CheckExternalLinksTask;
+use Symbiote\QueuedJobs\Services\QueuedJobService;
+use SilverStripe\PolyExecution\PolyOutput;
+
+class CMSExternalLinksController extends AdminController
+{
+    private static ?string $url_segment = 'externallinks';
+
+    private static string|array $required_permission_codes = [
+        'CMS_ACCESS_CMSMain',
+    ];
+
+    private static $allowed_actions = [
+        'getJobStatus',
+        'start'
+    ];
+
+    /**
+     * Respond to Ajax requests for info on a running job
+     */
+    public function getJobStatus(): HTTPResponse
+    {
+        $this->getResponse()->addHeader('X-Content-Type-Options', 'nosniff');
+        $track = BrokenExternalPageTrackStatus::get_latest();
+        if ($track) {
+            return $this->jsonSuccess(200, [
+                'TrackID' => $track->ID,
+                'Status' => $track->Status,
+                'Completed' => $track->getCompletedPages(),
+                'Total' => $track->getTotalPages()
+            ]);
+        }
+        return $this->jsonSuccess(200, []);
+    }
+
+    /**
+     * Starts a broken external link check
+     */
+    public function start(): HTTPResponse
+    {
+        // return if the a job is already running
+        $status = BrokenExternalPageTrackStatus::get_latest();
+        if ($status && $status->Status == 'Running') {
+            return $this->jsonSuccess(200, []);
+        }
+
+        // Create a new job
+        if (class_exists(QueuedJobService::class)) {
+            // Force the creation of a new run
+            BrokenExternalPageTrackStatus::create_status();
+            $checkLinks = new CheckExternalLinksJob();
+            singleton(QueuedJobService::class)->queueJob($checkLinks);
+        } else {
+            $task = CheckExternalLinksTask::create();
+            $task->runLinksCheck(PolyOutput::create(PolyOutput::FORMAT_HTML));
+        }
+        return $this->jsonSuccess(200, []);
+    }
+}