Skip to content

Commit

Permalink
feat(core): Add check gravatar image (reacherhq#1188)
Browse files Browse the repository at this point in the history
* feat(core): Add check gravatar image

* Add missing changes

* Fix compilation

* Fix tests

* Apply PR feedback

* Don't panic
  • Loading branch information
Agreon authored Oct 4, 2022
1 parent 0707bb4 commit 6a26035
Show file tree
Hide file tree
Showing 12 changed files with 180 additions and 30 deletions.
7 changes: 7 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,10 @@ The output will be a JSON with the below format, the fields should be self-expla
|| **Full inbox** | Is the inbox of this mailbox full? | `smtp.has_full_inbox` |
|| **Catch-all address** | Is this email address a [catch-all](https://debounce.io/blog/help/what-is-a-catch-all-or-accept-all/) address? | `smtp.is_catch_all` |
|| **Role account validation** | Is the email address a well-known role account? | `misc.is_role_account` |
|| **Gravatar Url** | The url of the [Gravatar](https://gravatar.com/) email address profile picture | `misc.gravatar_url` |
| 🔜 | **Free email provider check** | Is the email address bound to a known free email provider? | [Issue #89](https://github.com/reacherhq/check-if-email-exists/issues/89) |
| 🔜 | **Syntax validation, provider-specific** | According to the syntactic rules of the target mail provider, is the address syntactically valid? | [Issue #90](https://github.com/reacherhq/check-if-email-exists/issues/90) |
| 🔜 | **Honeypot detection** | Does email address under test hide a [honeypot](https://en.wikipedia.org/wiki/Spamtrap)? | [Issue #91](https://github.com/reacherhq/check-if-email-exists/issues/91) |
| 🔜 | **Gravatar** | Does this email address have a [Gravatar](https://gravatar.com/) profile picture? | [Issue #92](https://github.com/reacherhq/check-if-email-exists/issues/92) |
| 🔜 | **Have I Been Pwned?** | Has this email been compromised in a [data breach](https://haveibeenpwned.com/)? | [Issue #289](https://github.com/reacherhq/check-if-email-exists/issues/289) |

## 🤔 Why?
Expand Down
66 changes: 54 additions & 12 deletions backend/openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@
"is_reachable": "invalid",
"misc": {
"is_disposable": false,
"is_role_account": true
"is_role_account": true,
"gravatar_url": null
},
"mx": {
"accepts_mail": true,
Expand Down Expand Up @@ -109,7 +110,8 @@
"is_reachable": "invalid",
"misc": {
"is_disposable": false,
"is_role_account": true
"is_role_account": true,
"gravatar_url": null
},
"mx": {
"accepts_mail": true,
Expand Down Expand Up @@ -182,7 +184,14 @@
"$ref": "#/components/schemas/SyntaxDetails"
}
},
"required": ["input", "misc", "mx", "smtp", "syntax", "is_reachable"]
"required": [
"input",
"misc",
"mx",
"smtp",
"syntax",
"is_reachable"
]
},
"Error": {
"title": "Error",
Expand All @@ -198,7 +207,10 @@
"description": "A human-readable description of the error."
}
},
"required": ["type", "message"]
"required": [
"type",
"message"
]
},
"MiscDetails": {
"title": "MiscDetails",
Expand All @@ -212,9 +224,16 @@
"is_role_account": {
"type": "boolean",
"description": "Is this email a role-based account?"
},
"gravatar_url": {
"type": "string",
"description": "The Gravatar url of the image belonging to the given email."
}
},
"required": ["is_disposable", "is_role_account"]
"required": [
"is_disposable",
"is_role_account"
]
},
"MxDetails": {
"title": "MxDetails",
Expand All @@ -232,7 +251,10 @@
}
}
},
"required": ["accepts_mail", "records"],
"required": [
"accepts_mail",
"records"
],
"description": "Object holding the MX details of the mail server."
},
"SmtpDetails": {
Expand Down Expand Up @@ -261,7 +283,13 @@
"description": "Has this email address been disabled by the email provider?"
}
},
"required": ["can_connect_smtp", "has_full_inbox", "is_catch_all", "is_deliverable", "is_disabled"]
"required": [
"can_connect_smtp",
"has_full_inbox",
"is_catch_all",
"is_deliverable",
"is_disabled"
]
},
"SyntaxDetails": {
"title": "SyntaxDetails",
Expand All @@ -281,12 +309,21 @@
"description": "The username of the email, i.e. the part before the \"@\" symbol."
}
},
"required": ["domain", "is_valid_syntax", "username"]
"required": [
"domain",
"is_valid_syntax",
"username"
]
},
"Reachable": {
"type": "string",
"title": "Reachable",
"enum": ["invalid", "unknown", "safe", "risky"],
"enum": [
"invalid",
"unknown",
"safe",
"risky"
],
"description": "An enum to describe how confident we are that the recipient address is real: `safe`, `risky`, `invalid` and `unknown`. Check our FAQ to know the meanings of the 4 possibilities: https://help.reacher.email/email-attributes-inside-json."
},
"CheckEmailInput": {
Expand All @@ -310,7 +347,9 @@
"$ref": "#/components/schemas/CheckEmailInputProxy"
}
},
"required": ["to_email"]
"required": [
"to_email"
]
},
"CheckEmailInputProxy": {
"title": "CheckEmailInputProxy",
Expand All @@ -333,7 +372,10 @@
"description": "The proxy port."
}
},
"required": ["host", "port"]
"required": [
"host",
"port"
]
}
},
"securitySchemes": {
Expand All @@ -345,4 +387,4 @@
}
}
}
}
}
9 changes: 9 additions & 0 deletions backend/src/routes/bulk/results.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ struct JobResultCsvResponse {
misc_is_disposable: bool,
#[serde(rename = "misc.is_role_account")]
misc_is_role_account: bool,
#[serde(rename = "misc.gravatar_url")]
misc_gravatar_url: Option<String>,
#[serde(rename = "mx.accepts_mail")]
mx_accepts_mail: bool,
#[serde(rename = "smtp.can_connect")]
Expand Down Expand Up @@ -99,6 +101,7 @@ impl TryFrom<CsvWrapper> for JobResultCsvResponse {
let mut is_reachable: String = String::default();
let mut misc_is_disposable: bool = false;
let mut misc_is_role_account: bool = false;
let mut misc_gravatar_url: Option<String> = None;
let mut mx_accepts_mail: bool = false;
let mut smtp_can_connect: bool = false;
let mut smtp_has_full_inbox: bool = false;
Expand Down Expand Up @@ -136,6 +139,11 @@ impl TryFrom<CsvWrapper> for JobResultCsvResponse {
misc_is_role_account =
val.as_bool().ok_or("is_role_account should be a boolean")?
}
"gravatar_url" => {
if val.as_str() != None {
misc_gravatar_url = Some(val.to_string())
}
}
_ => {}
}
}
Expand Down Expand Up @@ -216,6 +224,7 @@ impl TryFrom<CsvWrapper> for JobResultCsvResponse {
is_reachable,
misc_is_disposable,
misc_is_role_account,
misc_gravatar_url,
mx_accepts_mail,
smtp_can_connect,
smtp_has_full_inbox,
Expand Down
4 changes: 2 additions & 2 deletions backend/tests/check_email.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ use serde_json;
use warp::http::StatusCode;
use warp::test::request;

const FOO_BAR_RESPONSE: &str = r#"{"input":"foo@bar","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":""}}"#;
const FOO_BAR_BAZ_RESPONSE: &str = r#"{"input":"[email protected]","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":"[email protected]","domain":"bar.baz","is_valid_syntax":true,"username":"foo"}}"#;
const FOO_BAR_RESPONSE: &str = r#"{"input":"foo@bar","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":""}}"#;
const FOO_BAR_BAZ_RESPONSE: &str = r#"{"input":"[email protected]","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":"[email protected]","domain":"bar.baz","is_valid_syntax":true,"username":"foo"}}"#;

#[tokio::test]
async fn test_input_foo_bar() {
Expand Down
3 changes: 3 additions & 0 deletions cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ ARGS:
<TO_EMAIL> The email to check

OPTIONS:
--check-gravatar <CHECK_GRAVATAR>
Whether to check for an existing gravatar image [env: CHECK_GRAVATAR=] [default: false]

--from-email <FROM_EMAIL>
The email to use in the `MAIL FROM:` SMTP command [env: FROM_EMAIL=] [default:
[email protected]]
Expand Down
7 changes: 6 additions & 1 deletion cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ pub struct Cli {
#[clap(long, env, default_value = "true", parse(try_from_str))]
pub yahoo_use_api: bool,

/// Whether to check if a gravatar image is existing for the given email.
#[clap(long, env, default_value = "false", parse(try_from_str))]
pub check_gravatar: bool,

/// The email to check.
pub to_email: String,
}
Expand All @@ -76,7 +80,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
.set_from_email(CONF.from_email.clone())
.set_hello_name(CONF.hello_name.clone())
.set_smtp_port(CONF.smtp_port)
.set_yahoo_use_api(CONF.yahoo_use_api);
.set_yahoo_use_api(CONF.yahoo_use_api)
.set_check_gravatar(CONF.check_gravatar);
if let Some(proxy_host) = &CONF.proxy_host {
input.set_proxy(CheckEmailInputProxy {
host: proxy_host.clone(),
Expand Down
3 changes: 2 additions & 1 deletion core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,13 @@ futures = { version = "0.3.24", optional = true }
fast-socks5 = "0.8.1"
log = "0.4.17"
mailchecker = "5.0.1"
rand = {version = "0.8.5", features = ["small_rng"] }
rand = { version = "0.8.5", features = ["small_rng"] }
regex = "1.6.0"
reqwest = { version = "0.11.11", features = ["json", "socks"] }
serde = { version = "1.0.145", features = ["derive"] }
serde_json = "1.0.85"
trust-dns-proto = "0.21.2"
md5 = "0.7.0"

[dev-dependencies]
tokio = { version = "1.21.2" }
Expand Down
61 changes: 61 additions & 0 deletions core/src/gravatar.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// check-if-email-exists
// Copyright (C) 2018-2022 Reacher

// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.

// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.

use crate::util::constants::LOG_TARGET;
use md5;
use md5::Digest;

const API_BASE_URL: &str = "https://www.gravatar.com/avatar/";

pub async fn check_gravatar(to_email: &str) -> Option<String> {
let client = reqwest::Client::new();

let mail_hash: Digest = md5::compute(to_email);

let url = format!("{}{:x}", API_BASE_URL, mail_hash);

log::debug!(
target: LOG_TARGET,
"[email={}] Request Gravatar API with url: {:?}",
to_email,
url
);

let response = client
.get(&url)
// This option is necessary to return a NotFound exception instead of the default gravatar
// image if none for the given email is found.
.query(&[("d", "404")])
.send()
.await;

log::debug!(
target: LOG_TARGET,
"[email={}] Gravatar response: {:?}",
to_email,
response
);

let response = match response {
Ok(response) => response,
Err(_) => return None,
};

match response.status() {
reqwest::StatusCode::OK => Some(url),
_ => None,
}
}
3 changes: 2 additions & 1 deletion core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
//! }
//! ```
pub mod gravatar;
pub mod misc;
pub mod mx;
pub mod smtp;
Expand Down Expand Up @@ -171,7 +172,7 @@ pub async fn check_email(input: &CheckEmailInput) -> CheckEmailOutput {
.collect::<Vec<String>>()
);

let my_misc = check_misc(&my_syntax);
let my_misc = check_misc(&my_syntax, input.check_gravatar).await;
log::debug!(
target: LOG_TARGET,
"[email={}] Found the following misc details: {:?}",
Expand Down
Loading

0 comments on commit 6a26035

Please sign in to comment.