feat: additional Gmail validation (reacherhq#1193)

* feat: additional Gmail validation

- check the validity of `gmail.com`/`googlemail.com` email addresses via
  the method outlined
  [here](https://blog.0day.rocks/abusing-gmail-to-get-previously-unlisted-e-mail-addresses-41544b62b2).
- run only via the `--gmail-use-api`/`gmail_use_api` flags (defaulting
  to `false`.)

relates reacherhq#937

* refactor: split out HTTP client

move the `create_client` method to a separate file; have the `yahoo` and
`gmail` modules reference this.

* test: add test for Gmail HTTP API

verify Gmail HTTP API behaviour with `[email protected]`, failure
indicating that the API is no longer reliable.

* fix: correct host checks for Gmail HTTP API

should check as per the MX host:

- for gmail.com or googlemail.com, this will look like
  `*.gmail-smtp-in.l.google.com.`.
- for Google Apps/Workspace domains, this will look like
  `*.aspmx.l.google.com.`.

Co-authored-by: PsypherPunk <[email protected]>

cli/README.md

@@ -62,6 +62,10 @@ OPTIONS:
         --yahoo-use-api <YAHOO_USE_API>
             For Yahoo email addresses, use Yahoo's API instead of connecting directly to their SMTP
             servers [env: YAHOO_USE_API=] [default: true]
+
+        --gmail-use-api <GMAIL_USE_API>
+            For Gmail email addresses, use Gmail's API instead of connecting directly to their SMTP
+            servers [env: GMAIL_USE_API=] [default: false]
 ```
 
 **💡 PRO TIP:** To show debug logs when running the binary, run:

cli/src/main.rs

@@ -58,6 +58,11 @@ pub struct Cli {
 	#[clap(long, env, default_value = "true", parse(try_from_str))]
 	pub yahoo_use_api: bool,
 
+	/// For Gmail email addresses, use Gmail's API instead of connecting
+	/// directly to their SMTP servers.
+	#[clap(long, env, default_value = "false", parse(try_from_str))]
+	pub gmail_use_api: bool,
+
 	/// Whether to check if a gravatar image is existing for the given email.
 	#[clap(long, env, default_value = "false", parse(try_from_str))]
 	pub check_gravatar: bool,
@@ -81,6 +86,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
 		.set_hello_name(CONF.hello_name.clone())
 		.set_smtp_port(CONF.smtp_port)
 		.set_yahoo_use_api(CONF.yahoo_use_api)
+		.set_gmail_use_api(CONF.gmail_use_api)
 		.set_check_gravatar(CONF.check_gravatar);
 	if let Some(proxy_host) = &CONF.proxy_host {
 		input.set_proxy(CheckEmailInputProxy {

core/src/smtp/connect.rs

@@ -352,6 +352,7 @@ pub async fn check_smtp_with_retry(
 		#[cfg(feature = "headless")]
 		Err(SmtpError::HotmailError(_)) => result,
 		Err(SmtpError::YahooError(_)) => result,
+		Err(SmtpError::GmailError(_)) => result,
 		// Only retry if the SMTP error was unknown.
 		Err(err) if err.get_description().is_none() => {
 			if count <= 1 {

core/src/smtp/error.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+use super::gmail::GmailError;
 #[cfg(feature = "headless")]
 use super::hotmail::HotmailError;
 use super::parser;
@@ -39,6 +40,8 @@ pub enum SmtpError {
 	TimeoutError(future::TimeoutError),
 	/// Error when verifying a Yahoo email via HTTP requests.
 	YahooError(YahooError),
+	/// Error when verifying a Gmail email via a HTTP request.
+	GmailError(GmailError),
 	/// Error when verifying a Hotmail email via headless browser.
 	#[cfg(feature = "headless")]
 	HotmailError(HotmailError),
@@ -62,6 +65,12 @@ impl From<YahooError> for SmtpError {
 	}
 }
 
+impl From<GmailError> for SmtpError {
+	fn from(e: GmailError) -> Self {
+		SmtpError::GmailError(e)
+	}
+}
+
 #[cfg(feature = "headless")]
 impl From<HotmailError> for SmtpError {
 	fn from(e: HotmailError) -> Self {

core/src/smtp/gmail.rs

@@ -0,0 +1,95 @@
+// check-if-email-exists
+// Copyright (C) 2018-2022 Reacher
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use super::SmtpDetails;
+use crate::{
+	smtp::http_api::create_client,
+	util::{
+		constants::LOG_TARGET, input_output::CheckEmailInput, ser_with_display::ser_with_display,
+	},
+};
+use async_smtp::EmailAddress;
+use reqwest::Error as ReqwestError;
+use serde::Serialize;
+use std::fmt;
+
+const GLXU_PAGE: &str = "https://mail.google.com/mail/gxlu";
+
+/// Possible errors when checking Gmail email addresses.
+#[derive(Debug, Serialize)]
+pub enum GmailError {
+	/// Error when serializing or deserializing HTTP requests and responses.
+	#[serde(serialize_with = "ser_with_display")]
+	ReqwestError(ReqwestError),
+}
+
+impl fmt::Display for GmailError {
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		write!(f, "{:?}", self)
+	}
+}
+
+impl From<ReqwestError> for GmailError {
+	fn from(error: ReqwestError) -> Self {
+		GmailError::ReqwestError(error)
+	}
+}
+
+/// Use HTTP request to verify if a Gmail email address exists.
+/// See: <https://blog.0day.rocks/abusing-gmail-to-get-previously-unlisted-e-mail-addresses-41544b62b2>
+pub async fn check_gmail(
+	to_email: &EmailAddress,
+	input: &CheckEmailInput,
+) -> Result<SmtpDetails, GmailError> {
+	let response = create_client(input, "gmail")?
+		.head(GLXU_PAGE)
+		.query(&[("email", to_email)])
+		.send()
+		.await?;
+
+	let email_exists = response.headers().contains_key("Set-Cookie");
+
+	log::debug!(
+		target: LOG_TARGET,
+		"[email={}] gmail response: {:?}",
+		to_email,
+		response
+	);
+
+	Ok(SmtpDetails {
+		can_connect_smtp: true,
+		is_deliverable: email_exists,
+		..Default::default()
+	})
+}
+
+#[cfg(test)]
+mod tests {
+	use std::str::FromStr;
+
+	use super::*;
+
+	#[tokio::test]
+	async fn should_return_is_deliverable_true() {
+		let to_email = EmailAddress::from_str("[email protected]").unwrap();
+		let input = CheckEmailInput::new("[email protected]".to_owned());
+
+		let smtp_details = check_gmail(&to_email, &input).await;
+
+		assert!(smtp_details.is_ok());
+		assert!(smtp_details.unwrap().is_deliverable);
+	}
+}

core/src/smtp/http_api.rs

@@ -0,0 +1,40 @@
+// check-if-email-exists
+// Copyright (C) 2018-2022 Reacher
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use crate::util::{constants::LOG_TARGET, input_output::CheckEmailInput};
+use reqwest::Error as ReqwestError;
+
+/// Helper function to create a reqwest client, with optional proxy.
+pub fn create_client(
+	input: &CheckEmailInput,
+	api_name: &str,
+) -> Result<reqwest::Client, ReqwestError> {
+	if let Some(proxy) = &input.proxy {
+		log::debug!(
+			target: LOG_TARGET,
+			"[email={}] Using proxy socks://{}:{} for {} API",
+			input.to_email,
+			proxy.host,
+			proxy.port,
+			api_name,
+		);
+
+		let proxy = reqwest::Proxy::all(&format!("socks5://{}:{}", proxy.host, proxy.port))?;
+		reqwest::Client::builder().proxy(proxy).build()
+	} else {
+		Ok(reqwest::Client::new())
+	}
+}

core/src/smtp/mod.rs

@@ -16,8 +16,10 @@
 
 mod connect;
 mod error;
+mod gmail;
 #[cfg(feature = "headless")]
 mod hotmail;
+mod http_api;
 mod parser;
 mod yahoo;
 
@@ -62,6 +64,11 @@ pub async fn check_smtp(
 			.await
 			.map_err(|err| err.into());
 	}
+	if input.gmail_use_api && host_lowercase.ends_with(".google.com.") {
+		return gmail::check_gmail(to_email, input)
+			.await
+			.map_err(|err| err.into());
+	}
 	#[cfg(feature = "headless")]
 	if let Some(webdriver) = &input.hotmail_use_headless {
 		if host_lowercase.contains("outlook") {

core/src/smtp/yahoo.rs

@@ -15,8 +15,11 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 use super::SmtpDetails;
-use crate::util::{
-	constants::LOG_TARGET, input_output::CheckEmailInput, ser_with_display::ser_with_display,
+use crate::{
+	smtp::http_api::create_client,
+	util::{
+		constants::LOG_TARGET, input_output::CheckEmailInput, ser_with_display::ser_with_display,
+	},
 };
 use async_smtp::EmailAddress;
 use regex::Regex;
@@ -105,31 +108,13 @@ impl From<SerdeError> for YahooError {
 	}
 }
 
-/// Helper function to create a reqwest client, with optional proxy.
-fn create_client(input: &CheckEmailInput) -> Result<reqwest::Client, ReqwestError> {
-	if let Some(proxy) = &input.proxy {
-		log::debug!(
-			target: LOG_TARGET,
-			"[email={}] Using proxy socks://{}:{} for Yahoo API",
-			input.to_email,
-			proxy.host,
-			proxy.port
-		);
-
-		let proxy = reqwest::Proxy::all(&format!("socks5://{}:{}", proxy.host, proxy.port))?;
-		reqwest::Client::builder().proxy(proxy).build()
-	} else {
-		Ok(reqwest::Client::new())
-	}
-}
-
 /// Use well-crafted HTTP requests to verify if a Yahoo email address exists.
 /// Inspired by https://github.com/hbattat/verifyEmail.
 pub async fn check_yahoo(
 	to_email: &EmailAddress,
 	input: &CheckEmailInput,
 ) -> Result<SmtpDetails, YahooError> {
-	let response = create_client(input)?
+	let response = create_client(input, "yahoo")?
 		.get(SIGNUP_PAGE)
 		.header("User-Agent", USER_AGENT)
 		.send()
@@ -178,7 +163,7 @@ pub async fn check_yahoo(
 	};
 
 	// Mimic a real HTTP request.
-	let response = create_client(input)?
+	let response = create_client(input, "yahoo")?
 		.post(SIGNUP_API)
 		.header("Origin", "https://login.yahoo.com")
 		.header("X-Requested-With", "XMLHttpRequest")

core/src/util/input_output.rs

@@ -90,6 +90,11 @@ pub struct CheckEmailInput {
 	///
 	/// Defaults to true.
 	pub yahoo_use_api: bool,
+	/// For Gmail email addresses, use Gmail's API instead of connecting
+	/// directly to their SMTP servers.
+	///
+	/// Defaults to false.
+	pub gmail_use_api: bool,
 	// Whether to check if a gravatar image is existing for the given email.
 	//
 	// Defaults to false
@@ -126,6 +131,7 @@ impl Default for CheckEmailInput {
 			smtp_security: SmtpSecurity::Opportunistic,
 			smtp_timeout: None,
 			yahoo_use_api: true,
+			gmail_use_api: false,
 			check_gravatar: false,
 			retries: 2,
 		}
@@ -234,6 +240,13 @@ impl CheckEmailInput {
 		self
 	}
 
+	/// Set whether to use Gmail's API or connecting directly to their SMTP
+	/// servers. Defaults to false.
+	pub fn set_gmail_use_api(&mut self, use_api: bool) -> &mut CheckEmailInput {
+		self.gmail_use_api = use_api;
+		self
+	}
+
 	/// Whether to check if a gravatar image is existing for the given email.
 	/// Defaults to false.
 	pub fn set_check_gravatar(&mut self, check_gravatar: bool) -> &mut CheckEmailInput {