Added APFS support #314

.pylintrc

@@ -7,7 +7,7 @@
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code
-extension-pkg-whitelist=pybde,pyewf,pyfsntfs,pyfvde,pyfwnt,pyqcow,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvshadow,pyvslvm
+extension-pkg-whitelist=pybde,pyewf,pyfsapfs,pyfsntfs,pyfvde,pyfwnt,pyqcow,pysigscan,pysmdev,pysmraw,pytsk3,pyvhdi,pyvmdk,pyvshadow,pyvslvm
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.

appveyor.yml

@@ -26,11 +26,11 @@ install:
 - cmd: if [%TARGET%]==[python27] (
     mkdir dependencies &&
     set PYTHONPATH=..\l2tdevtools &&
-    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track dev PyYAML dfdatetime dtfabric funcsigs libbde libewf libfsntfs libfvde libfwnt libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pycrypto pysqlite pytsk3 six )
+    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track dev PyYAML dfdatetime dtfabric funcsigs libbde libewf libfsapfs libfsntfs libfvde libfwnt libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pycrypto pysqlite pytsk3 six )
 - cmd: if [%TARGET%]==[python36] (
     mkdir dependencies &&
     set PYTHONPATH=..\l2tdevtools &&
-    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track dev PyYAML dfdatetime dtfabric funcsigs libbde libewf libfsntfs libfvde libfwnt libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pycrypto pytsk3 six )
+    "%PYTHON%\\python.exe" ..\l2tdevtools\tools\update.py --download-directory dependencies --machine-type %MACHINE_TYPE% --msi-targetdir "%PYTHON%" --track dev PyYAML dfdatetime dtfabric funcsigs libbde libewf libfsapfs libfsntfs libfvde libfwnt libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm mock pbr pycrypto pytsk3 six )
 
 build: off
 

config/dpkg/control

@@ -10,7 +10,7 @@ Homepage: https://github.com/log2timeline/dfvfs
 
 Package: python-dfvfs
 Architecture: all
-Depends: libbde-python (>= 20140531), libewf-python (>= 20131210), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20160418), libqcow-python (>= 20131204), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-backports.lzma, python-crypto (>= 2.6), python-dfdatetime (>= 20180324), python-dtfabric (>= 20170524), python-pysqlite2, python-pytsk3 (>= 20160721), python-yaml (>= 3.10), ${python:Depends}, ${misc:Depends}
+Depends: libbde-python (>= 20140531), libewf-python (>= 20131210), libfsapfs-python (>= 20181110), libfsntfs-python (>= 20151130), libfvde-python (>= 20160719), libfwnt-python (>= 20160418), libqcow-python (>= 20131204), libsigscan-python (>= 20150627), libsmdev-python (>= 20140529), libsmraw-python (>= 20140612), libvhdi-python (>= 20131210), libvmdk-python (>= 20140421), libvshadow-python (>= 20160109), libvslvm-python (>= 20160109), python-backports.lzma, python-crypto (>= 2.6), python-dfdatetime (>= 20181025), python-dtfabric (>= 20170524), python-pysqlite2, python-pytsk3 (>= 20160721), python-yaml (>= 3.10), ${python:Depends}, ${misc:Depends}
 Description: Python 2 module of dfVFS
  dfVFS, or Digital Forensics Virtual File System, provides read-only access to
  file-system objects from various storage media types and file formats. The goal
@@ -20,7 +20,7 @@ Description: Python 2 module of dfVFS
 
 Package: python3-dfvfs
 Architecture: all
-Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsntfs-python3 (>= 20151130), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libqcow-python3 (>= 20131204), libsigscan-python3 (>= 20150627), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-crypto (>= 2.6), python3-dfdatetime (>= 20180324), python3-dtfabric (>= 20170524), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
+Depends: libbde-python3 (>= 20140531), libewf-python3 (>= 20131210), libfsapfs-python3 (>= 20181110), libfsntfs-python3 (>= 20151130), libfvde-python3 (>= 20160719), libfwnt-python3 (>= 20160418), libqcow-python3 (>= 20131204), libsigscan-python3 (>= 20150627), libsmdev-python3 (>= 20140529), libsmraw-python3 (>= 20140612), libvhdi-python3 (>= 20131210), libvmdk-python3 (>= 20140421), libvshadow-python3 (>= 20160109), libvslvm-python3 (>= 20160109), python3-crypto (>= 2.6), python3-dfdatetime (>= 20181025), python3-dtfabric (>= 20170524), python3-pytsk3 (>= 20160721), python3-yaml (>= 3.10), ${python3:Depends}, ${misc:Depends}
 Description: Python 3 module of dfVFS
  dfVFS, or Digital Forensics Virtual File System, provides read-only access to
  file-system objects from various storage media types and file formats. The goal

config/linux/gift_copr_install.sh

@@ -11,6 +11,7 @@ set -e
 PYTHON2_DEPENDENCIES="PyYAML
                       libbde-python
                       libewf-python
+                      libfsapfs-python2
                       libfsntfs-python
                       libfvde-python
                       libfwnt-python
@@ -40,6 +41,8 @@ DEBUG_DEPENDENCIES="libbde-debuginfo
                     libbde-python-debuginfo
                     libewf-debuginfo
                     libewf-python-debuginfo
+                    libfsapfs-debuginfo
+                    libfsapfs-python2-debuginfo
                     libfsntfs-debuginfo
                     libfsntfs-python-debuginfo
                     libfvde-debuginfo

config/linux/gift_ppa_install.sh

@@ -10,6 +10,7 @@ set -e
 # This should not include packages only required for testing or development.
 PYTHON2_DEPENDENCIES="libbde-python
                       libewf-python
+                      libfsapfs-python
                       libfsntfs-python
                       libfvde-python
                       libfwnt-python
@@ -40,6 +41,8 @@ DEBUG_DEPENDENCIES="libbde-dbg
                     libbde-python-dbg
                     libewf-dbg
                     libewf-python-dbg
+                    libfsapfs-dbg
+                    libfsapfs-python-dbg
                     libfsntfs-dbg
                     libfsntfs-python-dbg
                     libfvde-dbg

config/linux/gift_ppa_install_py3.sh

@@ -10,6 +10,7 @@ set -e
 # This should not include packages only required for testing or development.
 PYTHON3_DEPENDENCIES="libbde-python3
                       libewf-python3
+                      libfsapfs-python3
                       libfsntfs-python3
                       libfvde-python3
                       libfwnt-python3
@@ -38,6 +39,8 @@ DEBUG_DEPENDENCIES="libbde-dbg
                     libbde-python3-dbg
                     libewf-dbg
                     libewf-python3-dbg
+                    libfsapfs-dbg
+                    libfsapfs-python3-dbg
                     libfsntfs-dbg
                     libfsntfs-python3-dbg
                     libfvde-dbg

config/travis/install.sh

@@ -5,15 +5,15 @@
 # This file is generated by l2tdevtools update-dependencies.py any dependency
 # related changes should be made in dependencies.ini.
 
-L2TBINARIES_DEPENDENCIES="PyYAML backports.lzma dfdatetime dtfabric libbde libewf libfsntfs libfvde libfwnt libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pycrypto pysqlite pytsk3";
+L2TBINARIES_DEPENDENCIES="PyYAML backports.lzma dfdatetime dtfabric libbde libewf libfsapfs libfsntfs libfvde libfwnt libqcow libsigscan libsmdev libsmraw libvhdi libvmdk libvshadow libvslvm pycrypto pysqlite pytsk3";
 
 L2TBINARIES_TEST_DEPENDENCIES="funcsigs mock pbr six";
 
-PYTHON2_DEPENDENCIES="libbde-python libewf-python libfsntfs-python libfvde-python libfwnt-python libqcow-python libsigscan-python libsmdev-python libsmraw-python libvhdi-python libvmdk-python libvshadow-python libvslvm-python python-backports.lzma python-crypto python-dfdatetime python-dtfabric python-pysqlite2 python-pytsk3 python-yaml";
+PYTHON2_DEPENDENCIES="libbde-python libewf-python libfsapfs-python libfsntfs-python libfvde-python libfwnt-python libqcow-python libsigscan-python libsmdev-python libsmraw-python libvhdi-python libvmdk-python libvshadow-python libvslvm-python python-backports.lzma python-crypto python-dfdatetime python-dtfabric python-pysqlite2 python-pytsk3 python-yaml";
 
 PYTHON2_TEST_DEPENDENCIES="python-coverage python-mock python-tox";
 
-PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-crypto python3-dfdatetime python3-dtfabric python3-pytsk3 python3-yaml";
+PYTHON3_DEPENDENCIES="libbde-python3 libewf-python3 libfsapfs-python3 libfsntfs-python3 libfvde-python3 libfwnt-python3 libqcow-python3 libsigscan-python3 libsmdev-python3 libsmraw-python3 libvhdi-python3 libvmdk-python3 libvshadow-python3 libvslvm-python3 python3-crypto python3-dfdatetime python3-dtfabric python3-pytsk3 python3-yaml";
 
 PYTHON3_TEST_DEPENDENCIES="python3-mock python3-setuptools python3-tox";
 

dependencies.ini

@@ -8,7 +8,7 @@ version_property: __version__
 
 [dfdatetime]
 dpkg_name: python-dfdatetime
-minimum_version: 20180324
+minimum_version: 20181025
 rpm_name: python-dfdatetime
 version_property: __version__
 
@@ -43,6 +43,14 @@ pypi_name: libewf-python
 rpm_name: libewf-python
 version_property: get_version()
 
+[pyfsapfs]
+dpkg_name: libfsapfs-python
+l2tbinaries_name: libfsapfs
+minimum_version: 20181110
+pypi_name: libfsapfs-python
+rpm_name: libfsapfs-python2
+version_property: get_version()
+
 [pyfsntfs]
 dpkg_name: libfsntfs-python
 l2tbinaries_name: libfsntfs

dfvfs/file_io/apfs_file_io.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+"""The APFS file-like object implementation."""
+
+from __future__ import unicode_literals
+
+import os
+
+from dfvfs.file_io import file_io
+from dfvfs.resolver import resolver
+
+
+class APFSFile(file_io.FileIO):
+  """File-like object using pyfsapfs."""
+
+  def __init__(self, resolver_context):
+    """Initializes a file-like object.
+
+    Args:
+      resolver_context (Context): resolver context.
+    """
+    super(APFSFile, self).__init__(resolver_context)
+    self._file_system = None
+    self._fsapfs_data_stream = None
+    self._fsapfs_file_entry = None
+
+  def _Close(self):
+    """Closes the file-like object."""
+    self._fsapfs_data_stream = None
+    self._fsapfs_file_entry = None
+
+    self._file_system.Close()
+    self._file_system = None
+
+  def _Open(self, path_spec=None, mode='rb'):
+    """Opens the file-like object defined by path specification.
+
+    Args:
+      path_spec (PathSpec): path specification.
+      mode (Optional[str]): file access mode.
+
+    Raises:
+      AccessError: if the access to open the file was denied.
+      IOError: if the file-like object could not be opened.
+      OSError: if the file-like object could not be opened.
+      PathSpecError: if the path specification is incorrect.
+      ValueError: if the path specification is invalid.
+    """
+    if not path_spec:
+      raise ValueError('Missing path specification.')
+
+    data_stream = getattr(path_spec, 'data_stream', None)
+
+    self._file_system = resolver.Resolver.OpenFileSystem(
+        path_spec, resolver_context=self._resolver_context)
+
+    file_entry = self._file_system.GetFileEntryByPathSpec(path_spec)
+    if not file_entry:
+      raise IOError('Unable to open file entry.')
+
+    fsapfs_data_stream = None
+    fsapfs_file_entry = file_entry.GetAPFSFileEntry()
+    if not fsapfs_file_entry:
+      raise IOError('Unable to open APFS file entry.')
+
+    if data_stream:
+      fsapfs_data_stream = fsapfs_file_entry.get_alternate_data_stream_by_name(
+          data_stream)
+      if not fsapfs_data_stream:
+        raise IOError('Unable to open data stream: {0:s}.'.format(
+            data_stream))
+
+    self._fsapfs_data_stream = fsapfs_data_stream
+    self._fsapfs_file_entry = fsapfs_file_entry
+
+  # Note: that the following functions do not follow the style guide
+  # because they are part of the file-like object interface.
+  # pylint: disable=invalid-name
+
+  def read(self, size=None):
+    """Reads a byte string from the file-like object at the current offset.
+
+    The function will read a byte string of the specified size or
+    all of the remaining data if no size was specified.
+
+    Args:
+      size (Optional[int]): number of bytes to read, where None is all
+          remaining data.
+
+    Returns:
+      bytes: data read.
+
+    Raises:
+      IOError: if the read failed.
+      OSError: if the read failed.
+    """
+    if not self._is_open:
+      raise IOError('Not opened.')
+
+    if self._fsapfs_data_stream:
+      return self._fsapfs_data_stream.read(size=size)
+    return self._fsapfs_file_entry.read(size=size)
+
+  def seek(self, offset, whence=os.SEEK_SET):
+    """Seeks to an offset within the file-like object.
+
+    Args:
+      offset (int): offset to seek to.
+      whence (Optional(int)): value that indicates whether offset is an absolute
+          or relative position within the file.
+
+    Raises:
+      IOError: if the seek failed.
+      OSError: if the seek failed.
+    """
+    if not self._is_open:
+      raise IOError('Not opened.')
+
+    if self._fsapfs_data_stream:
+      self._fsapfs_data_stream.seek(offset, whence)
+    else:
+      self._fsapfs_file_entry.seek(offset, whence)
+
+  def get_offset(self):
+    """Retrieves the current offset into the file-like object.
+
+    Return:
+      int: current offset into the file-like object.
+
+    Raises:
+      IOError: if the file-like object has not been opened.
+      OSError: if the file-like object has not been opened.
+    """
+    if not self._is_open:
+      raise IOError('Not opened.')
+
+    if self._fsapfs_data_stream:
+      return self._fsapfs_data_stream.get_offset()
+    return self._fsapfs_file_entry.get_offset()
+
+  def get_size(self):
+    """Retrieves the size of the file-like object.
+
+    Returns:
+      int: size of the file-like object data.
+
+    Raises:
+      IOError: if the file-like object has not been opened.
+      OSError: if the file-like object has not been opened.
+    """
+    if not self._is_open:
+      raise IOError('Not opened.')
+
+    if self._fsapfs_data_stream:
+      return self._fsapfs_data_stream.get_size()
+    return self._fsapfs_file_entry.get_size()

dfvfs/file_io/file_object_io.py

@@ -65,6 +65,7 @@ def _Open(self, path_spec=None, mode='rb'):
     if not self._file_object:
       raise IOError('Unable to open missing file-like object.')
 
+  # pylint: disable=redundant-returns-doc
   @abc.abstractmethod
   def _OpenFileObject(self, path_spec):
     """Opens the file-like object defined by path specification.

dfvfs/lib/apfs_container.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+"""Helper functions for APFS container support."""
+
+from __future__ import unicode_literals
+
+
+def APFSContainerPathSpecGetVolumeIndex(path_spec):
+  """Retrieves the volume index from the path specification.
+
+  Args:
+    path_spec (PathSpec): path specification.
+
+  Returns:
+    int: volume index or None if the index cannot be determined.
+  """
+  volume_index = getattr(path_spec, 'volume_index', None)
+  if volume_index is not None:
+    return volume_index
+
+  location = getattr(path_spec, 'location', None)
+  if location is None or not location.startswith('/apfs'):
+    return None
+
+  try:
+    volume_index = int(location[5:], 10) - 1
+  except (TypeError, ValueError):
+    volume_index = None
+
+  if volume_index is None or volume_index < 0 or volume_index > 99:
+    volume_index = None
+
+  return volume_index

dfvfs/lib/definitions.py

@@ -29,6 +29,8 @@
 ENCRYPTION_MODE_OFB = 'ofb'
 
 # The type indicator definitions.
+TYPE_INDICATOR_APFS = 'APFS'
+TYPE_INDICATOR_APFS_CONTAINER = 'APFS_CONTAINER'
 TYPE_INDICATOR_BDE = 'BDE'
 TYPE_INDICATOR_BZIP2 = 'BZIP2'
 TYPE_INDICATOR_COMPRESSED_STREAM = 'COMPRESSED_STREAM'
@@ -60,6 +62,7 @@
     TYPE_INDICATOR_FVDE])
 
 FILE_SYSTEM_TYPE_INDICATORS = frozenset([
+    TYPE_INDICATOR_APFS,
     TYPE_INDICATOR_NTFS,
     TYPE_INDICATOR_TSK])
 
@@ -71,6 +74,7 @@
     TYPE_INDICATOR_VMDK])
 
 VOLUME_SYSTEM_TYPE_INDICATORS = frozenset([
+    TYPE_INDICATOR_APFS_CONTAINER,
     TYPE_INDICATOR_LVM,
     TYPE_INDICATOR_TSK_PARTITION,
     TYPE_INDICATOR_VSHADOW])
@@ -91,6 +95,7 @@
 FILE_ENTRY_TYPE_LINK = 'link'
 FILE_ENTRY_TYPE_SOCKET = 'socket'
 FILE_ENTRY_TYPE_PIPE = 'pipe'
+FILE_ENTRY_TYPE_WHITEOUT = 'whiteout'
 
 # The format category definitions.
 FORMAT_CATEGORY_UNDEFINED = 0

dfvfs/lib/vshadow.py

@@ -24,7 +24,7 @@ def VShadowPathSpecGetStoreIndex(path_spec):
     store_index = None
     try:
       store_index = int(location[4:], 10) - 1
-    except ValueError:
+    except (TypeError, ValueError):
       pass
 
     if store_index is None or store_index < 0:

dfvfs/path/__init__.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 """Imports for path specification factory."""
 
+from dfvfs.path import apfs_container_path_spec
+from dfvfs.path import apfs_path_spec
 from dfvfs.path import bde_path_spec
 from dfvfs.path import compressed_stream_path_spec
 from dfvfs.path import cpio_path_spec