diff --git a/mix.exs b/mix.exs
index e41007f5..08e34dd0 100644
--- a/mix.exs
+++ b/mix.exs
@@ -8,9 +8,9 @@ defmodule Pow.MixProject do
       app: :pow,
       version: @version,
       elixir: "~> 1.6",
-      elixirc_paths: elixirc_paths(Mix.env()),
+      elixirc_paths: elixirc_paths(Mix.env(), optional_deps()),
       start_permanent: Mix.env() == :prod,
-      compilers: [:phoenix] ++ Mix.compilers(),
+      compilers: compilers(optional_deps()),
       deps: deps(),
 
       # Hex
@@ -35,9 +35,9 @@ defmodule Pow.MixProject do
 
   defp deps do
     [
-      {:ecto, "~> 2.2 or ~> 3.0"},
-      {:phoenix, "~> 1.3.0 or ~> 1.4.0"},
-      {:phoenix_html, ">= 2.0.0 and <= 3.0.0"},
+      {:ecto, "~> 2.2 or ~> 3.0", optional: true},
+      {:phoenix, "~> 1.3.0 or ~> 1.4.0", optional: true},
+      {:phoenix_html, ">= 2.0.0 and <= 3.0.0", optional: true},
       {:plug, ">= 1.5.0 and < 2.0.0", optional: true},
 
       {:phoenix_ecto, "~> 4.0.0", only: [:dev, :test]},
@@ -52,8 +52,53 @@ defmodule Pow.MixProject do
     ]
   end
 
-  defp elixirc_paths(:test), do: ["lib", "test/support"]
-  defp elixirc_paths(_), do: ["lib"]
+  def elixirc_paths(:test, _optional_deps), do: ["lib", "test/support"]
+  def elixirc_paths(_, optional_deps) do
+    case optional_deps_missing?(optional_deps) do
+      true -> paths_without_missing_optional_deps(optional_deps)
+      false  -> ["lib"]
+    end
+  end
+
+  def compilers(optional_deps) do
+    case phoenix_missing?(optional_deps) do
+      true  -> [:phoenix] ++ Mix.compilers
+      _     -> Mix.compilers()
+    end
+  end
+
+  defp phoenix_missing?(optional_deps) do
+    Keyword.get(optional_deps, :phoenix)
+  end
+
+  defp optional_deps_missing?(optional_deps) do
+    not Enum.empty?(optional_deps_missing(optional_deps))
+  end
+
+  defp optional_deps_missing(optional_deps) do
+    Enum.reject(optional_deps, &elem(&1, 1))
+  end
+
+  defp optional_deps do
+    for dep <- [:phoenix, :phoenix_html, :ecto, :plug] do
+      case Mix.ProjectStack.peek() do
+        %{config: config} -> {dep, Keyword.has_key?(config[:deps], dep)}
+        _                 -> {dep, true}
+      end
+    end
+  end
+
+  defp paths_without_missing_optional_deps(optional_deps) do
+    deps = optional_deps_missing(optional_deps)
+
+    "lib/**/*.ex"
+    |> Path.wildcard()
+    |> Enum.reject(&reject_deps_path?(deps, &1))
+  end
+
+  defp reject_deps_path?(deps, path) do
+    Enum.any?(deps, &String.contains?(path, "/#{elem(&1, 0)}"))
+  end
 
   defp package do
     [
diff --git a/test/mix_test.exs b/test/mix_test.exs
new file mode 100644
index 00000000..468d7d18
--- /dev/null
+++ b/test/mix_test.exs
@@ -0,0 +1,34 @@
+defmodule Pow.MixTest do
+  use ExUnit.Case
+
+  alias Pow.MixProject
+
+  test "elixirc_paths/1" do
+    assert MixProject.elixirc_paths(:test, []) == ["lib", "test/support"]
+    assert MixProject.elixirc_paths(:test, [phoenix: false]) == ["lib", "test/support"]
+
+    assert MixProject.elixirc_paths(:dev, []) == ["lib"]
+
+    assert paths = MixProject.elixirc_paths(:dev, [phoenix: false])
+    refute some_paths?(paths, "/phoenix")
+    assert some_paths?(paths, "/ecto")
+    assert some_paths?(paths, "/plug")
+
+    assert paths = MixProject.elixirc_paths(:dev, [ecto: false, plug: false])
+    assert some_paths?(paths, "/phoenix")
+    refute some_paths?(paths, "/ecto")
+    refute some_paths?(paths, "/plug")
+  end
+
+  test "compilers/1" do
+    compilers = [:yecc, :leex, :erlang, :elixir, :xref, :app]
+
+    assert MixProject.compilers([]) == compilers
+    assert MixProject.compilers([{:phoenix, false}]) == compilers
+    assert MixProject.compilers([{:phoenix, true}]) == [:phoenix] ++ compilers
+  end
+
+  defp some_paths?(paths, path) do
+    Enum.any?(paths, &(String.contains?(&1, path)))
+  end
+end