California Release (v1.64)

Please review the changes to this release in your site's /profiles/openasu/CHANGELOG.txt file.

CHANGELOG.txt

@@ -1,3 +1,19 @@
+Drupal 7.xx, xxxx-xx-xx (development version)
+-----------------------
+
+Drupal 7.65, 2019-03-20
+-----------------------
+- Fixed security issues:
+   - SA-CORE-2019-004
+
+Drupal 7.64, 2019-02-06
+-----------------------
+- [regression] Unset the 'host' header in drupal_http_request() during redirect
+- Fixed: 7.x does not have Phar protection and Phar tests are failing on Drupal 7
+- Fixed: Notice: Undefined index: display_field in file_field_widget_value() (line 582 of /module/file/file.field.inc)
+- Performance improvement: Registry rebuild should not parse the same file twice in the same request
+- Fixed _registry_update() to clear caches after transaction is committed
+
 Drupal 7.63, 2019-01-16
 -----------------------
 - Fixed a fatal error for some Drush users introduced by SA-CORE-2019-002.

includes/bootstrap.inc

@@ -8,7 +8,7 @@
 /**
  * The current system version.
  */
-define('VERSION', '7.63');
+define('VERSION', '7.65');
 
 /**
  * Core API compatibility.

includes/common.inc

@@ -1094,6 +1094,11 @@ function drupal_http_request($url, array $options = array()) {
       elseif ($options['max_redirects']) {
         // Redirect to the new location.
         $options['max_redirects']--;
+
+        // We need to unset the 'Host' header
+        // as we are redirecting to a new location.
+        unset($options['headers']['Host']);
+
         $result = drupal_http_request($location, $options);
         $result->redirect_code = $code;
       }

includes/file.inc

@@ -997,16 +997,22 @@ function file_build_uri($path) {
  * @return
  *   The destination filepath, or FALSE if the file already exists
  *   and FILE_EXISTS_ERROR is specified.
+ *
+ * @throws RuntimeException
+ *   Thrown if the filename contains invalid UTF-8.
  */
 function file_destination($destination, $replace) {
+  $basename = drupal_basename($destination);
+  if (!drupal_validate_utf8($basename)) {
+    throw new RuntimeException(sprintf("Invalid filename '%s'", $basename));
+  }
   if (file_exists($destination)) {
     switch ($replace) {
       case FILE_EXISTS_REPLACE:
         // Do nothing here, we want to overwrite the existing file.
         break;
 
       case FILE_EXISTS_RENAME:
-        $basename = drupal_basename($destination);
         $directory = drupal_dirname($destination);
         $destination = file_create_filename($basename, $directory);
         break;
@@ -1222,11 +1228,20 @@ function file_unmunge_filename($filename) {
  * @return
  *   File path consisting of $directory and a unique filename based off
  *   of $basename.
+ *
+ * @throws RuntimeException
+ *   Thrown if the $basename is not valid UTF-8 or another error occurs
+ *   stripping control characters.
  */
 function file_create_filename($basename, $directory) {
+  $original = $basename;
   // Strip control characters (ASCII value < 32). Though these are allowed in
   // some filesystems, not many applications handle them well.
   $basename = preg_replace('/[\x00-\x1F]/u', '_', $basename);
+  if (preg_last_error() !== PREG_NO_ERROR) {
+    throw new RuntimeException(sprintf("Invalid filename '%s'", $original));
+  }
+
   if (substr(PHP_OS, 0, 3) == 'WIN') {
     // These characters are not allowed in Windows filenames
     $basename = str_replace(array(':', '*', '?', '"', '<', '>', '|'), '_', $basename);
@@ -1567,7 +1582,13 @@ function file_save_upload($form_field_name, $validators = array(), $destination
   if (substr($destination, -1) != '/') {
     $destination .= '/';
   }
-  $file->destination = file_destination($destination . $file->filename, $replace);
+  try {
+    $file->destination = file_destination($destination . $file->filename, $replace);
+  }
+  catch (RuntimeException $e) {
+    drupal_set_message(t('The file %source could not be uploaded because the name is invalid.', array('%source' => $form_field_name)), 'error');
+    return FALSE;
+  }
   // If file_destination() returns FALSE then $replace == FILE_EXISTS_ERROR and
   // there's an existing file so we need to bail.
   if ($file->destination === FALSE) {
@@ -2134,9 +2155,33 @@ function file_download_access($uri) {
  *   'filename', and 'name' members corresponding to the matching files.
  */
 function file_scan_directory($dir, $mask, $options = array(), $depth = 0) {
+  // Default nomask option.
+  $nomask = '/(\.\.?|CVS)$/';
+
+  // Overrides the $nomask variable accordingly if $options['nomask'] is set.
+  //
+  // Allow directories specified in settings.php to be ignored. You can use this
+  // to not check for files in common special-purpose directories. For example,
+  // node_modules and bower_components. Ignoring irrelevant directories is a
+  // performance boost.
+  if (!isset($options['nomask'])) {
+    $ignore_directories = variable_get(
+      'file_scan_ignore_directories',
+      array()
+    );
+
+    foreach ($ignore_directories as $index => $ignore_directory) {
+      $ignore_directories[$index] = preg_quote($ignore_directory, '/');
+    }
+
+    if (!empty($ignore_directories)) {
+      $nomask = '/^(\.\.?)|CVS|' . implode('|', $ignore_directories) . '$/';
+    }
+  }
+
   // Merge in defaults.
   $options += array(
-    'nomask' => '/(\.\.?|CVS)$/',
+    'nomask' => $nomask,
     'callback' => 0,
     'recurse' => TRUE,
     'key' => 'uri',

includes/registry.inc

@@ -19,7 +19,6 @@
  * Does the work for registry_update().
  */
 function _registry_update() {
-
   // The registry serves as a central autoloader for all classes, including
   // the database query builders. However, the registry rebuild process
   // requires write ability to the database, which means having access to the
@@ -33,6 +32,11 @@ function _registry_update() {
   require_once DRUPAL_ROOT . '/includes/database/select.inc';
   require_once DRUPAL_ROOT . '/includes/database/' . $driver . '/query.inc';
 
+  // During the first registry rebuild in a request, we check all the files.
+  // During subsequent rebuilds, we only add new files. It makes the rebuilding
+  // process faster during installation of modules.
+  static $check_existing_files = TRUE;
+
   // Get current list of modules and their files.
   $modules = db_query("SELECT * FROM {system} WHERE type = 'module'")->fetchAll();
   // Get the list of files we are going to parse.
@@ -55,6 +59,9 @@ function _registry_update() {
     $files["$filename"] = array('module' => '', 'weight' => 0);
   }
 
+  // Initialize an empty array for the unchanged files.
+  $unchanged_files = array();
+
   $transaction = db_transaction();
   try {
     // Allow modules to manually modify the list of files before the registry
@@ -63,10 +70,19 @@ function _registry_update() {
     // list can then be added to the list of files that the registry will parse,
     // or modify attributes of a file.
     drupal_alter('registry_files', $files, $modules);
+
     foreach (registry_get_parsed_files() as $filename => $file) {
       // Add the hash for those files we have already parsed.
       if (isset($files[$filename])) {
-        $files[$filename]['hash'] = $file['hash'];
+        if ($check_existing_files === TRUE) {
+          $files[$filename]['hash'] = $file['hash'];
+        }
+        else {
+          // Ignore that file for this request, it has been parsed previously
+          // and it is unlikely it has changed.
+          unset($files[$filename]);
+          $unchanged_files[$filename] = $file;
+        }
       }
       else {
         // Flush the registry of resources in files that are no longer on disc
@@ -79,8 +95,12 @@ function _registry_update() {
           ->execute();
       }
     }
+
     $parsed_files = _registry_parse_files($files);
 
+    // Add unchanged files to the files.
+    $files += $unchanged_files;
+
     $unchanged_resources = array();
     $lookup_cache = array();
     if ($cache = cache_get('lookup_cache', 'cache_bootstrap')) {
@@ -89,19 +109,24 @@ function _registry_update() {
     foreach ($lookup_cache as $key => $file) {
       // If the file for this cached resource is carried over unchanged from
       // the last registry build, then we can safely re-cache it.
-      if ($file && in_array($file, array_keys($files)) && !in_array($file, $parsed_files)) {
+      if ($file && isset($files[$file]) && !in_array($file, $parsed_files, TRUE)) {
         $unchanged_resources[$key] = $file;
       }
     }
-    module_implements('', FALSE, TRUE);
-    _registry_check_code(REGISTRY_RESET_LOOKUP_CACHE);
   }
   catch (Exception $e) {
     $transaction->rollback();
     watchdog_exception('registry', $e);
     throw $e;
   }
 
+  module_implements('', FALSE, TRUE);
+  _registry_check_code(REGISTRY_RESET_LOOKUP_CACHE);
+
+  // During the next run in this request, don't bother re-checking existing
+  // files.
+  $check_existing_files = FALSE;
+
   // We have some unchanged resources, warm up the cache - no need to pay
   // for looking them up again.
   if (count($unchanged_resources) > 0) {

modules/field/modules/number/number.test

@@ -69,7 +69,7 @@ class NumberFieldTestCase extends DrupalWebTestCase {
     preg_match('|test-entity/manage/(\d+)/edit|', $this->url, $match);
     $id = $match[1];
     $this->assertRaw(t('test_entity @id has been created.', array('@id' => $id)), 'Entity was created');
-    $this->assertRaw(round($value, 2), 'Value is displayed.');
+    $this->assertRaw($value, 'Value is displayed.');
 
     // Try to create entries with more than one decimal separator; assert fail.
     $wrong_entries = array(

modules/file/file.field.inc

@@ -599,7 +599,7 @@ function file_field_widget_value($element, $input = FALSE, $form_state) {
     // If the display field is present make sure its unchecked value is saved.
     $field = field_widget_field($element, $form_state);
     if (empty($input['display'])) {
-      $input['display'] = $field['settings']['display_field'] ? 0 : 1;
+      $input['display'] = !empty($field['settings']['display_field']) ? 0 : 1;
     }
   }
 

modules/file/tests/file.test

@@ -1875,3 +1875,60 @@ class FileFieldAnonymousSubmission extends FileFieldTestCase {
   }
 
 }
+
+/**
+ * Tests the file_scan_directory() function.
+ */
+class FileScanDirectory extends FileFieldTestCase {
+
+  /**
+   * @var string
+   */
+  protected $path;
+
+  /**
+   * {@inheritdoc}
+   */
+  public static function getInfo() {
+    return array(
+      'name' => 'File ScanDirectory',
+      'description' => 'Tests the file_scan_directory() function.',
+      'group' => 'File',
+    );
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  function setUp() {
+    parent::setUp();
+
+    $this->path = 'modules/file/tests/fixtures/file_scan_ignore';
+  }
+
+  /**
+   * Tests file_scan_directory() obeys 'file_scan_ignore_directories' setting.
+   * If nomask is not passed as argument, it should use the default settings.
+   * If nomask is passed as argument, it should obey this rule.
+   */
+  public function testNoMask() {
+    $files = file_scan_directory($this->path, '/\.txt$/');
+    $this->assertEqual(3, count($files), '3 text files found when not ignoring directories.');
+
+    global $conf;
+    $conf['file_scan_ignore_directories'] = array('frontend_framework');
+
+    $files = file_scan_directory($this->path, '/\.txt$/');
+    $this->assertEqual(1, count($files), '1 text files found when ignoring directories called "frontend_framework".');
+
+    // Make that directories specified by default still work when a new nomask is provided.
+    $files = file_scan_directory($this->path, '/\.txt$/', array('nomask' => '/^c.txt/'));
+    $this->assertEqual(2, count($files), '2 text files found when an "nomask" option is passed in.');
+
+    // Ensure that the directories in file_scan_ignore_directories are escaped using preg_quote.
+    $conf['file_scan_ignore_directories'] = array('frontend.*');
+    $files = file_scan_directory($this->path, '/\.txt$/');
+    $this->assertEqual(3, count($files), '2 text files found when ignoring a directory that is not there.');
+  }
+
+}

modules/simpletest/drupal_web_test_case.php

@@ -3012,7 +3012,7 @@ protected function assertRaw($raw, $message = '', $group = 'Other') {
     if (!$message) {
       $message = t('Raw "@raw" found', array('@raw' => $raw));
     }
-    return $this->assert(strpos($this->drupalGetContent(), $raw) !== FALSE, $message, $group);
+    return $this->assert(strpos($this->drupalGetContent(), (string) $raw) !== FALSE, $message, $group);
   }
 
   /**
@@ -3032,7 +3032,7 @@ protected function assertNoRaw($raw, $message = '', $group = 'Other') {
     if (!$message) {
       $message = t('Raw "@raw" not found', array('@raw' => $raw));
     }
-    return $this->assert(strpos($this->drupalGetContent(), $raw) === FALSE, $message, $group);
+    return $this->assert(strpos($this->drupalGetContent(), (string) $raw) === FALSE, $message, $group);
   }
 
   /**

modules/simpletest/tests/bootstrap.test

@@ -729,16 +729,12 @@ class BootstrapMiscTestCase extends DrupalUnitTestCase {
    * Tests that the drupal_check_memory_limit() function works as expected.
    */
   function testCheckMemoryLimit() {
-    $memory_limit = ini_get('memory_limit');
     // Test that a very reasonable amount of memory is available.
     $this->assertTrue(drupal_check_memory_limit('30MB'), '30MB of memory tested available.');
 
-    // Get the available memory and multiply it by two to make it unreasonably
-    // high.
-    $twice_avail_memory = ($memory_limit * 2) . 'MB';
-
+    // Test an unlimited memory limit.
     // The function should always return true if the memory limit is set to -1.
-    $this->assertTrue(drupal_check_memory_limit($twice_avail_memory, -1), 'drupal_check_memory_limit() returns TRUE when a limit of -1 (none) is supplied');
+    $this->assertTrue(drupal_check_memory_limit('9999999999YB', -1), 'drupal_check_memory_limit() returns TRUE when a limit of -1 (none) is supplied');
 
     // Test that even though we have 30MB of memory available - the function
     // returns FALSE when given an upper limit for how much memory can be used.

modules/simpletest/tests/file.test

@@ -957,6 +957,15 @@ class FileDirectoryTest extends FileTestCase {
     $path = file_create_filename($basename, $directory);
     $this->assertEqual($path, $expected, format_string('Creating a new filepath from %original equals %new.', array('%new' => $path, '%original' => $original)), 'File');
 
+    try {
+      $filename = "a\xFFtest\x80€.txt";
+      file_create_filename($filename, $directory);
+      $this->fail('Expected exception not thrown');
+    }
+    catch (RuntimeException $e) {
+      $this->assertEqual("Invalid filename '$filename'", $e->getMessage());
+    }
+
     // @TODO: Finally we copy a file into a directory several times, to ensure a properly iterating filename suffix.
   }
 
@@ -989,6 +998,14 @@ class FileDirectoryTest extends FileTestCase {
     $this->assertNotEqual($path, $destination, 'A new filepath destination is created when filepath destination already exists with FILE_EXISTS_RENAME.', 'File');
     $path = file_destination($destination, FILE_EXISTS_ERROR);
     $this->assertEqual($path, FALSE, 'An error is returned when filepath destination already exists with FILE_EXISTS_ERROR.', 'File');
+
+    try {
+      file_destination("core/misc/a\xFFtest\x80€.txt", FILE_EXISTS_REPLACE);
+      $this->fail('Expected exception not thrown');
+    }
+    catch (RuntimeException $e) {
+      $this->assertEqual("Invalid filename 'a\xFFtest\x80€.txt'", $e->getMessage());
+    }
   }
 
   /**

modules/system/system.install

@@ -2885,7 +2885,7 @@ function system_update_7061(&$sandbox) {
     if (!db_table_exists('system_update_7061')) {
       $table = array(
         'description' => t('Stores temporary data for system_update_7061.'),
-        'fields' => array('vid' => array('type' => 'int')),
+        'fields' => array('vid' => array('type' => 'int', 'not null' => TRUE)),
         'primary key' => array('vid'),
       );
       db_create_table('system_update_7061', $table);