Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

217 api validation b #225

Open
wants to merge 15 commits into
base: master
Choose a base branch
from
Open
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ vendor/
docs/_build/
docs/html/
docs/doxyxml
.vscode
74 changes: 74 additions & 0 deletions includes/api_rate_limit.inc
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
<?php

/**
* Utility/timer functions for API Rate Limiting
*/

/**
* Utility function to get the SSE and microseconds
*/
function get_mtime() {
$right_now = DateTime::createFromFormat('U.u', microtime(true));
// Occasionally calling DateTime will fail, keep trying until it doesn't.
while(!$right_now)
{
usleep(50000);
$right_now = DateTime::createFromFormat('U.u', microtime(true));
}
return floatval($right_now->format("U.u"));
}

/**
* Adds an entry to the global variable that an NCBI request has been made
*/
function tripal_eutils_register_access() {
$last_access = variable_get('tripal_eutils_api_timer',NULL);

$last_access = get_mtime();

variable_set('tripal_eutils_api_timer',$last_access);
}

/**
* Checks that EUtils can continue making requests to NCBI without
* exceeding the API rate limits
*
* @return boolean
* TRUE if EUtils has to wait, false otherwise
* Enough time has passed or no recent requests made
*/
function api_timer_wait() {
$last_access = variable_get('tripal_eutils_api_timer',NULL);
// Determine the rate limit
if (variable_get('tripal_eutils_ncbi_api_key'))
{
// API Key exists, was checked at previous check. Full speed ahead
$limit = 10;
}
else {
// No API key exists. Rate limit to 3 per second
$limit = 3;
}

// No known previous access, go for it
if (!$last_access) {
return false;
}
else {
// Determine appropriate time to wait (+ a little padding, NCBI is sensitive)
$wait_time = 1.0/$limit + 0.2;

// Current time (format: seconds since epoch.microseconds)
$now = get_mtime();

$variance = number_format($now - $last_access, 5);
if ($variance > $wait_time) {
// Don't wait
return false;
}
else {
// Gotta wait
return true;
}
}
}
2 changes: 1 addition & 1 deletion includes/resources/EFetch.inc
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class EFetch extends EUtilsRequest {
$this->addParam('db', $db);

$api_key = variable_get('tripal_eutils_ncbi_api_key');
if ($api_key) {
if ($api_key && $this->api_key_validate($api_key)) {
$this->addParam('api_key', $api_key);
}
}
Expand Down
2 changes: 1 addition & 1 deletion includes/resources/ESearch.inc
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class ESearch extends EUtilsRequest {
$this->setBaseURL('https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi');
$this->addParam('db', $db);
$api_key = variable_get('tripal_eutils_ncbi_api_key');
if ($api_key) {
if ($api_key && $this->api_key_validate($api_key)) {
$this->addParam('api_key', $api_key);
}
}
Expand Down
2 changes: 1 addition & 1 deletion includes/resources/ESummary.inc
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
$this->setBaseURL('https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi');
$this->addParam('db', $db);
$api_key = variable_get('tripal_eutils_ncbi_api_key');
if ($api_key) {
if ($api_key && $this->api_key_validate($api_key)) {
$this->addParam('api_key', $api_key);
}
}
Expand Down
10 changes: 7 additions & 3 deletions includes/resources/EUtils.inc
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,13 @@ class EUtils {
$provider->addParam('retmode', 'xml');
}

// TODO: global static timer.
// That pauses if we're passing our alloted queries/second.
// Wait until timer has accumulated enough time to continue
while (api_timer_wait()){}

$response = $provider->get();
// Request was made, register this timestamp for API rate limit
tripal_eutils_register_access();

$this->checkResponseSuccess($response, $db, $accession);
$xml = $response->xml();

Expand Down Expand Up @@ -259,4 +263,4 @@ class EUtils {
}
}

}
}
57 changes: 57 additions & 0 deletions includes/resources/EUtilsRequest.inc
Original file line number Diff line number Diff line change
Expand Up @@ -170,4 +170,61 @@ class EUtilsRequest {
return substr($str, 0, strlen($with)) === $with;
}

/**
* Check whether the API key is still valid
*
* @param string $api_key
* @param string $db_url
*
* @return bool
*/
protected function api_key_validate($api_key) {
// Get the value of tripal_eutils_recent_validation [last_validation, value]
$recent_validation = variable_get('tripal_eutils_recent_validation',NULL);
if ($recent_validation)
{
$now = intval(date("u"));
// At some point, the API key was positively validated
if ($recent_validation[1])
{
// Test if expired
if ($now - $recent_validation[1] > 1800)
{
return false;
}
else
{
return true;
}
}
// The API key was most recently found to be invalid
else {
return true;
}
}
else // No known API validations or API validation has expired
{
$now = intval(date("u"));
$test_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi";
$test_url = $test_url . '?api_key=' . $api_key;

// Make a request to NCBI, collect and parse the response
$response = drupal_http_request($test_url);

if (array_key_exists('error',$response)) {
// Some error exists with the API key. Print this
// Set the recent
variable_set('tripal_eutils_recent_validation',[$now,FALSE]);
// and return false
return false;
}
else {
// Set the recent
variable_set('tripal_eutils_recent_validation',[$now,TRUE]);
return true;
}

}
}

}
65 changes: 35 additions & 30 deletions includes/tripal_euitils_admin_settings.form.inc
Original file line number Diff line number Diff line change
Expand Up @@ -31,51 +31,56 @@ function tripal_eutils_admin_settings_form($form, &$form_state) {
* Implements hook_validate().
*/
function tripal_eutils_admin_settings_form_validate($form, &$form_state) {
// Check to see if there exists an API key
if ($form_state['values']['api_key']) {
// Prepare the URL and parameters (with API key)
$url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi';
$url .= '?api_key='.$form_state['values']['api_key'];

// TODO We would validate the key here. Perhaps make a test query?
// Make the request
$response = drupal_http_request($url);

// Prepare the URL and parameters (with API key)
$url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi';
$url .= '?api_key='.$form_state['values']['api_key'];

// Make the request
$response = drupal_http_request($url);

// Error handling
if (array_key_exists('error',$response))
{
if(preg_match('/\b(API key invalid)\b/',$response->data))
{
//drupal_set_message("Invalid API Key",'error');
form_set_error('api_key',t("Invalid API Key"));
}
else if(preg_match('/\b(API key not wellformed)\b/',$response->data))
// Error handling
if (array_key_exists('error',$response))
{
//drupal_set_message("Unknown or malformed API Key",'error');
form_set_error('api_key',t("Unknown or malformed API Key"));
}
else
{
drupal_set_message("Other unknown error:".$response->data,'error');
form_set_error('api_key',t("Unknown issue with API key"));
if(preg_match('/\b(API key invalid)\b/',$response->data))
{
//drupal_set_message("Invalid API Key",'error');
form_set_error('api_key',t("Invalid API Key"));
}
else if(preg_match('/\b(API key not wellformed)\b/',$response->data))
{
//drupal_set_message("Unknown or malformed API Key",'error');
form_set_error('api_key',t("Unknown or malformed API Key"));
}
else
{
drupal_set_message("Other unknown error:".$response->data,'error');
form_set_error('api_key',t("Unknown issue with API key"));
}
}
// No apparent errors. Check that NCBI responded with the correct number of
// requests per second that API key users should have (currently 10)
}
// No apparent errors. Check that NCBI responded with the correct number of
// requests per second that API key users should have (currently 10)

}

/**
* Implements hook_submit().
*/
function tripal_eutils_admin_settings_form_submit($form, &$form_state) {
if (isset($form_state['values']['api_key'])) {
// Valid API key
if (!empty($form_state['values']['api_key'])) {
$api_key = $form_state['values']['api_key'];
variable_set('tripal_eutils_ncbi_api_key', $api_key);
$now = $now = intval(date("u"));
variable_set('tripal_eutils_recent_validation', [$now, TRUE]);
drupal_set_message("API Key validated and saved.");
}
// No API key
else {
variable_set('tripal_eutils_ncbi_api_key', NULL);
variable_set('tripal_eutils_recent_validation', NULL);
drupal_set_message("API Key removed.");
}

drupal_set_message("API Key validated and saved.");

}
1 change: 1 addition & 0 deletions tripal_eutils.module
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ require_once 'includes/repositories/EUtilsPubmedRepository.inc';

// Repository helpers. Assist with looking up terms.
require_once 'includes/TagMapper.inc';
require_once 'includes/api_rate_limit.inc';

// Formatters.
require_once 'includes/formatters/EUtilsFormatter.inc';
Expand Down