Merge Develop into Master

LinkCrawler/LinkCrawler.Tests/LinkCrawler.Tests.csproj

@@ -30,6 +30,15 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="AutoFixture, Version=4.5.0.0, Culture=neutral, PublicKeyToken=b24654c590009d4f, processorArchitecture=MSIL">
+      <HintPath>..\packages\AutoFixture.4.5.0\lib\net452\AutoFixture.dll</HintPath>
+    </Reference>
+    <Reference Include="AutoFixture.AutoMoq, Version=4.5.0.0, Culture=neutral, PublicKeyToken=b24654c590009d4f, processorArchitecture=MSIL">
+      <HintPath>..\packages\AutoFixture.AutoMoq.4.5.0\lib\net452\AutoFixture.AutoMoq.dll</HintPath>
+    </Reference>
+    <Reference Include="Fare, Version=2.1.0.0, Culture=neutral, PublicKeyToken=ea68d375bf33a7c8, processorArchitecture=MSIL">
+      <HintPath>..\packages\Fare.2.1.1\lib\net35\Fare.dll</HintPath>
+    </Reference>
     <Reference Include="Moq, Version=4.2.1510.2205, Culture=neutral, PublicKeyToken=69f491c39445e920, processorArchitecture=MSIL">
       <HintPath>..\packages\Moq.4.2.1510.2205\lib\net40\Moq.dll</HintPath>
       <Private>True</Private>
@@ -39,6 +48,7 @@
       <Private>True</Private>
     </Reference>
     <Reference Include="System" />
+    <Reference Include="System.ComponentModel.DataAnnotations" />
     <Reference Include="System.Core" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
@@ -50,23 +60,30 @@
   <ItemGroup>
     <Compile Include="LinkCrawlerTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="UtilsTests\ClientsTests\SlackClientTests.cs" />
     <Compile Include="UtilsTests\ExtensionsTests\HttpStatusCodeExtensionsTests.cs" />
     <Compile Include="UtilsTests\ExtensionsTests\StringExtensionsTests.cs" />
-    <Compile Include="UtilsTests\HelpersTests\ValidUrlParserTests.cs" />
+    <Compile Include="UtilsTests\OutputsTests\SlackOutputTests.cs" />
+    <Compile Include="UtilsTests\ParsersTests\ValidUrlParserTests.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="..\LinkCrawler\App.config">
       <Link>App.config</Link>
     </None>
     <None Include="packages.config" />
   </ItemGroup>
-  <ItemGroup />
+  <ItemGroup>
+    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\LinkCrawler\LinkCrawler.csproj">
       <Project>{db53303b-f9fb-4d77-b656-d05db0420e6a}</Project>
       <Name>LinkCrawler</Name>
     </ProjectReference>
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="UtilsTests\HelpersTests\" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

LinkCrawler/LinkCrawler.Tests/LinkCrawlerTests.cs

@@ -5,6 +5,7 @@
 using LinkCrawler.Utils.Settings;
 using Moq;
 using NUnit.Framework;
+using System.Linq;
 
 namespace LinkCrawler.Tests
 {
@@ -49,7 +50,7 @@ public void CrawlForLinksInResponse_ResponseModelWithMarkup_ValidUrlFoundInMarku
             mockResponseModel.Setup(x => x.Markup).Returns(markup);
 
             LinkCrawler.CrawlForLinksInResponse(mockResponseModel.Object);
-            Assert.That(LinkCrawler.VisitedUrlList.Contains(url));
+            Assert.That(LinkCrawler.UrlList.Where(l=>l.Address == url).Count() > 0);
         }
     }
 }

LinkCrawler/LinkCrawler.Tests/UtilsTests/ClientsTests/SlackClientTests.cs

@@ -0,0 +1,38 @@
+using LinkCrawler.Utils.Clients;
+using NUnit.Framework;
+using LinkCrawler.Utils.Settings;
+
+namespace LinkCrawler.Tests.UtilsTests.ClientsTests {
+
+    [TestFixture]
+    public class SlackClientTests {
+
+        //MethodName_StateUnderTest_ExpectedBehaviour
+        [Test]
+        public void SlackClient_InstantiationWithWebHookUrl_InstantiatedCorrectlyWithWebHookUrl() {
+            MockSettings settings = new MockSettings(true);
+            SlackClient sc = new SlackClient(settings);
+
+            Assert.AreEqual(@"https://hooks.slack.com/services/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", sc.WebHookUrl);
+            Assert.AreEqual("Homer Bot", sc.BotName);
+            Assert.AreEqual(":homer:", sc.BotIcon);
+            Assert.AreEqual("*Doh! There is a link not working* Url: {0} Statuscode: {1} The link is placed on this page: {2}", sc.MessageFormat);
+            Assert.IsTrue(sc.HasWebHookUrl);
+
+        }
+
+        [Test]
+        public void SlackClient_InstantiationWithoutWebHookUrl_InstantiatedCorrectlyWithoutWebHookUrl() {
+            MockSettings settings = new MockSettings(false);
+            SlackClient sc = new SlackClient(settings);
+
+            Assert.AreEqual("", sc.WebHookUrl);
+            Assert.AreEqual("Homer Bot", sc.BotName);
+            Assert.AreEqual(":homer:", sc.BotIcon);
+            Assert.AreEqual("*Doh! There is a link not working* Url: {0} Statuscode: {1} The link is placed on this page: {2}", sc.MessageFormat);
+            Assert.IsFalse(sc.HasWebHookUrl);
+
+        }
+    }
+
+}

LinkCrawler/LinkCrawler.Tests/UtilsTests/ExtensionsTests/RegexExtensionsTest.cs

@@ -0,0 +1,23 @@
+using LinkCrawler.Utils.Extensions;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace LinkCrawler.Tests.UtilsTests.ExtensionsTests
+{
+    [TestFixture]
+    public class RegexExtensionsTest
+    {
+        public void IsNotMatch_Should_Return_False()
+        {
+
+            string regex = "(^http[s]?:\\/{2})|(^www)|(^\\/{1,2})";
+            string url = "website.com:///podcast/";
+            bool expression = RegexExtensions.IsNotMatch(new System.Text.RegularExpressions.Regex(regex), url);
+            Assert.IsFalse(expression);
+        }
+    }
+}

LinkCrawler/LinkCrawler.Tests/UtilsTests/ExtensionsTests/StringExtensionsTests.cs

@@ -118,5 +118,49 @@ public void StartsWithIgnoreCase_DifferentLetterAndSameCase_True()
             var result = word.StartsWithIgnoreCase(letter);
             Assert.AreEqual(false, result);
         }
+
+        [Test]
+        public void TrimEnd_InputNull_Null()
+        {
+            string input = null;
+            string expected = null;
+
+            var actual = input.TrimEnd("");
+
+            Assert.AreEqual(expected, actual);
+        }
+
+        [Test]
+        public void TrimEnd_InputEndsWithSuffix_RemovesSuffix()
+        {
+            string input = "friend";
+            string expected = "fri";
+
+            var actual = input.TrimEnd("end");
+
+            Assert.AreEqual(expected, actual);
+        }
+
+        [Test]
+        public void TrimEnd_InputEndsWithSuffixDifferentCase_ReturnsOriginal()
+        {
+            string input = "friEND";
+            string expected = "friEND";
+
+            var actual = input.TrimEnd("end");
+
+            Assert.AreEqual(expected, actual);
+        }
+
+        [Test]
+        public void TrimEnd_InputEndsWithSuffixDifferentCase_ReturnsEmptyString()
+        {
+            string input = "friend";
+            string expected = string.Empty;
+
+            var actual = input.TrimEnd("friend");
+
+            Assert.AreEqual(expected, actual);
+        }
     }
 }

LinkCrawler/LinkCrawler.Tests/UtilsTests/OutputsTests/SlackOutputTests.cs

@@ -0,0 +1,39 @@
+using AutoFixture;
+using NUnit.Framework;
+using LinkCrawler.Utils.Outputs;
+using AutoFixture.AutoMoq;
+using LinkCrawler.Utils.Clients;
+using LinkCrawler.Models;
+using Moq;
+
+namespace LinkCrawler.Tests.UtilsTests.OutputsTests
+{
+    [TestFixture]
+    public class SlackOutputTests
+    {
+        private Fixture _fixture;
+
+        [SetUp]
+        public void SetUup()
+        {
+            _fixture = new Fixture();
+            _fixture.Customize(new AutoMoqCustomization());
+        }
+
+        [Test]
+        public void WriteError()
+        {
+
+            var slackClient = _fixture.Freeze<Mock<ISlackClient>>();
+            var slackOutput = _fixture.Freeze<SlackOutput>();
+            var responseModel = _fixture.Create<IResponseModel>();
+
+            slackClient.Setup(x => x.NotifySlack(responseModel));
+
+            slackOutput.WriteError(responseModel);
+
+            slackClient.VerifyAll();
+
+        }
+    }
+}

LinkCrawler/LinkCrawler.Tests/UtilsTests/HelpersTests/ValidUrlParserTests.cs → LinkCrawler/LinkCrawler.Tests/UtilsTests/ParsersTests/ValidUrlParserTests.cs

@@ -2,7 +2,7 @@
 using LinkCrawler.Utils.Settings;
 using NUnit.Framework;
 
-namespace LinkCrawler.Tests.UtilsTests.HelpersTests
+namespace LinkCrawler.Tests.UtilsTests.ParsersTests
 {
     [TestFixture]
     public class ValidUrlParserTests

LinkCrawler/LinkCrawler.Tests/packages.config

@@ -1,5 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="AutoFixture" version="4.5.0" targetFramework="net452" />
+  <package id="AutoFixture.AutoMoq" version="4.5.0" targetFramework="net452" />
+  <package id="Fare" version="2.1.1" targetFramework="net452" />
   <package id="Moq" version="4.2.1510.2205" targetFramework="net452" />
   <package id="NUnit" version="3.0.1" targetFramework="net452" />
 </packages>

LinkCrawler/LinkCrawler/App.config

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8" ?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <configSections>
     <section name="outputProviders" type="System.Configuration.DictionarySectionHandler" />
@@ -13,20 +13,26 @@
     <add key="SuccessHttpStatusCodes" value="1xx,2xx,3xx" />
     <!--explanation of regex below: http://regexr.com/3cqt9 -->
     <add key="ValidUrlRegex" value="(^http[s]?:\/{2})|(^www)|(^\/{1,2})" />
+    <add key="PrintSummary" value="false" />
     <add key="Csv.FilePath" value="C:\tmp\output.csv" />
     <add key="Csv.Overwrite" value="true" />
     <add key="Csv.Delimiter" value=";" />
     <add key="Slack.WebHook.Url" value="https://hooks.slack.com/services/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" />
     <add key="Slack.WebHook.Bot.Name" value="Homer Bot" />
     <add key="Slack.WebHook.Bot.IconEmoji" value=":homer:" />
-    <add key="Slack.WebHook.Bot.MessageFormat" value="*Doh! There is a link not working*
-Url: {0}
-Statuscode: {1}
-The link is placed on this page: {2}" />
+    <add key="Slack.WebHook.Bot.MessageFormat" value="*Doh! There is a link not working* Url: {0} Statuscode: {1} The link is placed on this page: {2}" />
   </appSettings>
   <outputProviders>
     <add key="Console" value="LinkCrawler.Utils.Outputs.ConsoleOutput" />
     <!--<add key="Csv" value="LinkCrawler.Utils.Outputs.CsvOutput" />-->
     <!--<add key="Slack" value="LinkCrawler.Utils.Outputs.SlackOutput" />-->
   </outputProviders>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Moq" publicKeyToken="69f491c39445e920" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.2.1510.2205" newVersion="4.2.1510.2205" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
 </configuration>

LinkCrawler/LinkCrawler/LinkCrawler.cs

@@ -7,7 +7,8 @@
 using RestSharp;
 using System;
 using System.Collections.Generic;
-using LinkCrawler.Utils.Outputs;
+using System.Diagnostics;
+using System.Linq;
 
 namespace LinkCrawler
 {
@@ -19,23 +20,27 @@ public class LinkCrawler
         public IEnumerable<IOutput> Outputs { get; set; }
         public IValidUrlParser ValidUrlParser { get; set; }
         public bool OnlyReportBrokenLinksToOutput { get; set; }
-        public static List<string> VisitedUrlList { get; set; }
+        public static List<LinkModel> UrlList;
         private ISettings _settings;
+        private Stopwatch _timer;
 
         public LinkCrawler(IEnumerable<IOutput> outputs, IValidUrlParser validUrlParser, ISettings settings)
         {
             BaseUrl = settings.BaseUrl;
             Outputs = outputs;
             ValidUrlParser = validUrlParser;
             CheckImages = settings.CheckImages;
-            VisitedUrlList = new List<string>();
+            UrlList = new List<LinkModel>();
             RestRequest = new RestRequest(Method.GET).SetHeader("Accept", "*/*");
             OnlyReportBrokenLinksToOutput = settings.OnlyReportBrokenLinksToOutput;
             _settings = settings;
+            _timer = new Stopwatch();
         }
 
         public void Start()
         {
+            _timer.Start();
+            UrlList.Add(new LinkModel(BaseUrl));
             SendRequest(BaseUrl);
         }
 
@@ -68,10 +73,13 @@ public void CrawlForLinksInResponse(IResponseModel responseModel)
 
             foreach (var url in linksFoundInMarkup)
             {
-                if (VisitedUrlList.Contains(url))
-                    continue;
+                lock (UrlList)
+                {
+                    if (UrlList.Where(l => l.Address == url).Count() > 0)
+                        continue;
 
-                VisitedUrlList.Add(url);
+                    UrlList.Add(new LinkModel(url));
+                }
                 SendRequest(url, responseModel.RequestedUrl);
             }
         }
@@ -92,6 +100,54 @@ public void WriteOutput(IResponseModel responseModel)
                     output.WriteInfo(responseModel);
                 }
             }
+
+            CheckIfFinal(responseModel);
+        }
+
+        private void CheckIfFinal(IResponseModel responseModel)
+        {
+            lock (UrlList)
+            {
+
+                // First set the status code for the completed link (this will set "CheckingFinished" to true)
+                foreach (LinkModel lm in UrlList.Where(l => l.Address == responseModel.RequestedUrl))
+                {
+                    lm.StatusCode = responseModel.StatusCodeNumber;
+                }
+
+                // Then check to see whether there are any pending links left to check
+                if ((UrlList.Count > 1) && (UrlList.Where(l => l.CheckingFinished == false).Count() == 0))
+                {
+                    FinaliseSession();
+                }
+            }
+        }
+
+        private void FinaliseSession()
+        {
+            _timer.Stop();
+            if (_settings.PrintSummary)
+            {
+                List<string> messages = new List<string>();
+                messages.Add(""); // add blank line to differentiate summary from main output
+
+                messages.Add("Processing complete. Checked " + UrlList.Count() + " links in " + _timer.ElapsedMilliseconds.ToString() + "ms");
+
+                messages.Add("");
+                messages.Add(" Status | # Links");
+                messages.Add(" -------+--------");
+
+                IEnumerable<IGrouping<int, string>> StatusSummary = UrlList.GroupBy(link => link.StatusCode, link => link.Address);
+                foreach(IGrouping<int,string> statusGroup in StatusSummary)
+                {
+                    messages.Add(String.Format("   {0}  | {1,5}", statusGroup.Key, statusGroup.Count()));
+                }
+
+                foreach (var output in Outputs)
+                {
+                    output.WriteInfo(messages.ToArray());
+                }
+            }
         }
     }
 }