Changes to expose data stream extents #597 (#631)

log2timeline · Jan 13, 2022 · 7b70305 · 7b70305
1 parent ef47f46
commit 7b70305
Show file tree

Hide file tree

Showing 11 changed files with 181 additions and 91 deletions.
diff --git a/appveyor.yml b/appveyor.yml
@@ -17,7 +17,7 @@ environment:
     HOMEBREW_NO_INSTALL_CLEANUP: 1
 
 install:
-- cmd: "%PYTHON%\\python.exe -m pip install -U pip setuptools wheel"
+- cmd: "%PYTHON%\\python.exe -m pip install -U pip setuptools twine wheel"
 - cmd: "%PYTHON%\\python.exe -m pip install pywin32 WMI"
 - cmd: "%PYTHON%\\python.exe %PYTHON%\\Scripts\\pywin32_postinstall.py -install"
 - ps: If ($isWindows) { .\config\appveyor\install.ps1 }

diff --git a/config/dpkg/control b/config/dpkg/control
@@ -9,7 +9,7 @@ Homepage: https://github.com/log2timeline/dfvfs
 
 Package: python3-dfvfs
 Architecture: all
-Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20210721), libfshfs-python3 (>= 20210722), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20210726), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20200101), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220110), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20211113), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
+Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20201107), libfsext-python3 (>= 20220112), libfshfs-python3 (>= 20210722), libfsntfs-python3 (>= 20211229), libfsxfs-python3 (>= 20210726), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20210717), libluksde-python3 (>= 20200101), libmodi-python3 (>= 20210405), libphdi-python3 (>= 20220110), libqcow-python3 (>= 20201213), libsigscan-python3 (>= 20191221), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20201014), libvmdk-python3 (>= 20140421), libvsgpt-python3 (>= 20211115), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-cffi-backend (>= 1.9.1), python3-cryptography (>= 2.0.2), python3-dfdatetime (>= 20211113), python3-dtfabric (>= 20170524), python3-idna (>= 2.5), python3-pytsk3 (>= 20210419), python3-pyxattr (>= 0.7.2), python3-yaml (>= 3.10), ${misc:Depends}
 Description: Python 3 module of dfVFS
  dfVFS, or Digital Forensics Virtual File System, provides read-only access to
  file-system objects from various storage media types and file formats. The goal

diff --git a/dependencies.ini b/dependencies.ini
@@ -58,7 +58,7 @@ version_property: get_version()
 [pyfsext]
 dpkg_name: libfsext-python3
 l2tbinaries_name: libfsext
-minimum_version: 20210721
+minimum_version: 20220112
 pypi_name: libfsext-python
 rpm_name: libfsext-python3
 version_property: get_version()

diff --git a/dfvfs/vfs/ext_file_entry.py b/dfvfs/vfs/ext_file_entry.py
@@ -9,6 +9,7 @@
 from dfvfs.vfs import attribute
 from dfvfs.vfs import ext_attribute
 from dfvfs.vfs import ext_directory
+from dfvfs.vfs import extent
 from dfvfs.vfs import file_entry
 
 
@@ -244,6 +245,30 @@ def size(self):
     """int: size of the file entry in bytes or None if not available."""
     return self._fsext_file_entry.size
 
+  def GetExtents(self, data_stream_name=''):
+    """Retrieves extents of a specific data stream.
+
+    Returns:
+      list[Extent]: extents of the data stream.
+    """
+    extents = []
+    if (self.entry_type == definitions.FILE_ENTRY_TYPE_FILE and
+        not data_stream_name):
+      for extent_index in range(self._fsext_file_entry.number_of_extents):
+        extent_offset, extent_size, extent_flags = (
+            self._fsext_file_entry.get_extent(extent_index))
+
+        if extent_flags & 0x1:
+          extent_type = definitions.EXTENT_TYPE_SPARSE
+        else:
+          extent_type = definitions.EXTENT_TYPE_DATA
+
+        data_stream_extent = extent.Extent(
+            extent_type=extent_type, offset=extent_offset, size=extent_size)
+        extents.append(data_stream_extent)
+
+    return extents
+
   def GetEXTFileEntry(self):
     """Retrieves the EXT file entry.
 

diff --git a/dfvfs/vfs/ntfs_file_entry.py b/dfvfs/vfs/ntfs_file_entry.py
@@ -239,7 +239,21 @@ def GetExtents(self, data_stream_name=''):
       list[Extent]: extents of the data stream.
     """
     extents = []
-    if data_stream_name:
+    if not data_stream_name:
+      for extent_index in range(self._fsntfs_file_entry.number_of_extents):
+        extent_offset, extent_size, extent_flags = (
+            self._fsntfs_file_entry.get_extent(extent_index))
+
+        if extent_flags & 0x1:
+          extent_type = definitions.EXTENT_TYPE_SPARSE
+        else:
+          extent_type = definitions.EXTENT_TYPE_DATA
+
+        data_stream_extent = extent.Extent(
+            extent_type=extent_type, offset=extent_offset, size=extent_size)
+        extents.append(data_stream_extent)
+
+    else:
       fsntfs_data_stream = (
           self._fsntfs_file_entry.get_alternate_data_stream_by_name(
               data_stream_name))
@@ -258,20 +272,6 @@ def GetExtents(self, data_stream_name=''):
               extent_type=extent_type, offset=extent_offset, size=extent_size)
           extents.append(data_stream_extent)
 
-    else:
-      for extent_index in range(self._fsntfs_file_entry.number_of_extents):
-        extent_offset, extent_size, extent_flags = (
-            self._fsntfs_file_entry.get_extent(extent_index))
-
-        if extent_flags & 0x1:
-          extent_type = definitions.EXTENT_TYPE_SPARSE
-        else:
-          extent_type = definitions.EXTENT_TYPE_DATA
-
-        data_stream_extent = extent.Extent(
-            extent_type=extent_type, offset=extent_offset, size=extent_size)
-        extents.append(data_stream_extent)
-
     return extents
 
   def GetFileObject(self, data_stream_name=''):

diff --git a/dfvfs/vfs/tsk_file_entry.py b/dfvfs/vfs/tsk_file_entry.py
@@ -703,7 +703,7 @@ def GetExtents(self, data_stream_name=''):
 
     Raises:
       BackEndError: if pytsk3 returns a non UTF-8 formatted name or no file
-          system block size.
+          system block size or data stream size.
     """
     data_pytsk_attribute = None
     for pytsk_attribute in self._tsk_file:
@@ -723,8 +723,9 @@ def GetExtents(self, data_stream_name=''):
 
         # The data stream is returned as a name-less attribute of type
         # pytsk3.TSK_FS_ATTR_TYPE_DEFAULT.
-        if (attribute_type == pytsk3.TSK_FS_ATTR_TYPE_DEFAULT and not name and
-            not data_stream_name):
+        if (self.entry_type == definitions.FILE_ENTRY_TYPE_FILE and
+            attribute_type == pytsk3.TSK_FS_ATTR_TYPE_DEFAULT and
+            not name and not data_stream_name):
           data_pytsk_attribute = pytsk_attribute
           break
 
@@ -740,19 +741,40 @@ def GetExtents(self, data_stream_name=''):
       if not block_size:
         raise errors.BackEndError('pytsk3 returned no file system block size.')
 
+      data_stream_size = getattr(data_pytsk_attribute.info, 'size', None)
+      if data_stream_size is None:
+        raise errors.BackEndError('pytsk3 returned no data stream size.')
+
+      data_stream_number_of_blocks, remainder = divmod(
+          data_stream_size, block_size)
+      if remainder:
+        data_stream_number_of_blocks += 1
+
+      total_number_of_blocks = 0
       for pytsk_attr_run in data_pytsk_attribute:
         if pytsk_attr_run.flags & pytsk3.TSK_FS_ATTR_RUN_FLAG_SPARSE:
           extent_type = definitions.EXTENT_TYPE_SPARSE
         else:
           extent_type = definitions.EXTENT_TYPE_DATA
 
         extent_offset = pytsk_attr_run.addr * block_size
-        extent_size = pytsk_attr_run.len * block_size
+        extent_size = pytsk_attr_run.len
+
+        # Note that the attribute data runs can be larger than the actual
+        # allocated size.
+        if total_number_of_blocks + extent_size > data_stream_number_of_blocks:
+          extent_size = data_stream_number_of_blocks - total_number_of_blocks
+
+        total_number_of_blocks += extent_size
+        extent_size *= block_size
 
         data_stream_extent = extent.Extent(
             extent_type=extent_type, offset=extent_offset, size=extent_size)
         extents.append(data_stream_extent)
 
+        if total_number_of_blocks >= data_stream_number_of_blocks:
+          break
+
     return extents
 
   def GetFileObject(self, data_stream_name=''):

diff --git a/requirements.txt b/requirements.txt
@@ -7,7 +7,7 @@ dtfabric >= 20170524
 libbde-python >= 20140531
 libewf-python >= 20131210
 libfsapfs-python >= 20201107
-libfsext-python >= 20210721
+libfsext-python >= 20220112
 libfshfs-python >= 20210722
 libfsntfs-python >= 20211229
 libfsxfs-python >= 20210726

diff --git a/setup.cfg b/setup.cfg
@@ -20,7 +20,7 @@ build_requires = python3-setuptools
 requires = libbde-python3 >= 20140531
            libewf-python3 >= 20131210
            libfsapfs-python3 >= 20201107
-           libfsext-python3 >= 20210721
+           libfsext-python3 >= 20220112
            libfshfs-python3 >= 20210722
            libfsntfs-python3 >= 20211229
            libfsxfs-python3 >= 20210726