Package oauth2client :: Module locked_file |
[hide private]
[frames] | no frames]

# Source Code for Module oauth2client.locked_file

      1  # Copyright 2011 Google Inc.
      2  #
      3  # Licensed under the Apache License, Version 2.0 (the "License");
      4  # you may not use this file except in compliance with the License.
      5  # You may obtain a copy of the License at
      6  #
      7  #      http://www.apache.org/licenses/LICENSE-2.0
      8  #
      9  # Unless required by applicable law or agreed to in writing, software
     10  # distributed under the License is distributed on an "AS IS" BASIS,
     11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     12  # See the License for the specific language governing permissions and
     13  # limitations under the License.
     14
     15  """Locked file interface that should work on Unix and Windows pythons.
     16
     17  This module first tries to use fcntl locking to ensure serialized access
     18  to a file, then falls back on a lock file if that is unavialable.
     19
     20  Usage:
     21      f = LockedFile('filename', 'r+b', 'rb')
     22      f.open_and_lock()
     23      if f.is_locked():
     24        print 'Acquired filename with r+b mode'
     25        f.file_handle().write('locked data')
     26      else:
     27        print 'Aquired filename with rb mode'
     28      f.unlock_and_close()
     29  """
     30
     31  __author__ = 'cache@google.com (David T McWherter)'
     32
     33  import errno
     34  import logging
     35  import os
     36  import time
     37
     38  from oauth2client import util
     39
     40  logger = logging.getLogger(__name__)
     41
     42
     43 -class CredentialsFileSymbolicLinkError(Exception):
     44    """Credentials files must not be symbolic links."""
     45
     46
     47 -class AlreadyLockedException(Exception):
     48    """Trying to lock a file that has already been locked by the LockedFile."""
     49    pass
     50
     51
     52 -def validate_file(filename):
     53    if os.path.islink(filename):
     54      raise CredentialsFileSymbolicLinkError(
     55          'File: %s is a symbolic link.' % filename)
     56
     57 -class _Opener(object):
     58    """Base class for different locking primitives."""
     59
     60 -  def __init__(self, filename, mode, fallback_mode):
     61      """Create an Opener.
     62
     63      Args:
     64        filename: string, The pathname of the file.
     65        mode: string, The preferred mode to access the file with.
     66        fallback_mode: string, The mode to use if locking fails.
     67      """
     68      self._locked = False
     69      self._filename = filename
     70      self._mode = mode
     71      self._fallback_mode = fallback_mode
     72      self._fh = None
     73
     74 -  def is_locked(self):
     75      """Was the file locked."""
     76      return self._locked
     77
     78 -  def file_handle(self):
     79      """The file handle to the file. Valid only after opened."""
     80      return self._fh
     81
     82 -  def filename(self):
     83      """The filename that is being locked."""
     84      return self._filename
     85
     86 -  def open_and_lock(self, timeout, delay):
     87      """Open the file and lock it.
     88
     89      Args:
     90        timeout: float, How long to try to lock for.
     91        delay: float, How long to wait between retries.
     92      """
     93      pass
     94
     95 -  def unlock_and_close(self):
     96      """Unlock and close the file."""
     97      pass
     98
     99
    100 -class _PosixOpener(_Opener):
    101    """Lock files using Posix advisory lock files."""
    102
    103 -  def open_and_lock(self, timeout, delay):
    104      """Open the file and lock it.
    105
    106      Tries to create a .lock file next to the file we're trying to open.
    107
    108      Args:
    109        timeout: float, How long to try to lock for.
    110        delay: float, How long to wait between retries.
    111
    112      Raises:
    113        AlreadyLockedException: if the lock is already acquired.
    114        IOError: if the open fails.
    115        CredentialsFileSymbolicLinkError if the file is a symbolic link.
    116      """
    117      if self._locked:
    118        raise AlreadyLockedException('File %s is already locked' %
    119                                     self._filename)
    120      self._locked = False
    121
    122      validate_file(self._filename)
    123      try:
    124        self._fh = open(self._filename, self._mode)
    125      except IOError, e:
    126        # If we can't access with _mode, try _fallback_mode and don't lock.
    127        if e.errno == errno.EACCES:
    128          self._fh = open(self._filename, self._fallback_mode)
    129          return
    130
    131      lock_filename = self._posix_lockfile(self._filename)
    132      start_time = time.time()
    133      while True:
    134        try:
    135          self._lock_fd = os.open(lock_filename,
    136                                  os.O_CREAT|os.O_EXCL|os.O_RDWR)
    137          self._locked = True
    138          break
    139
    140        except OSError, e:
    141          if e.errno != errno.EEXIST:
    142            raise
    143          if (time.time() - start_time) >= timeout:
    144            logger.warn('Could not acquire lock %s in %s seconds' % (
    145                lock_filename, timeout))
    146            # Close the file and open in fallback_mode.
    147            if self._fh:
    148              self._fh.close()
    149            self._fh = open(self._filename, self._fallback_mode)
    150            return
    151          time.sleep(delay)
    152
    153 -  def unlock_and_close(self):
    154      """Unlock a file by removing the .lock file, and close the handle."""
    155      if self._locked:
    156        lock_filename = self._posix_lockfile(self._filename)
    157        os.close(self._lock_fd)
    158        os.unlink(lock_filename)
    159        self._locked = False
    160        self._lock_fd = None
    161      if self._fh:
    162        self._fh.close()
    163
    164 -  def _posix_lockfile(self, filename):
    165      """The name of the lock file to use for posix locking."""
    166      return '%s.lock' % filename
    167
    168
    169  try:
    170    import fcntl
    171
    172 -  class _FcntlOpener(_Opener):
    173      """Open, lock, and unlock a file using fcntl.lockf."""
    174
    175 -    def open_and_lock(self, timeout, delay):
    176        """Open the file and lock it.
    177
    178        Args:
    179          timeout: float, How long to try to lock for.
    180          delay: float, How long to wait between retries
    181
    182        Raises:
    183          AlreadyLockedException: if the lock is already acquired.
    184          IOError: if the open fails.
    185          CredentialsFileSymbolicLinkError if the file is a symbolic link.
    186        """
    187        if self._locked:
    188          raise AlreadyLockedException('File %s is already locked' %
    189                                       self._filename)
    190        start_time = time.time()
    191
    192        validate_file(self._filename)
    193        try:
    194          self._fh = open(self._filename, self._mode)
    195        except IOError, e:
    196          # If we can't access with _mode, try _fallback_mode and don't lock.
    197          if e.errno == errno.EACCES:
    198            self._fh = open(self._filename, self._fallback_mode)
    199            return
    200
    201        # We opened in _mode, try to lock the file.
    202        while True:
    203          try:
    204            fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX)
    205            self._locked = True
    206            return
    207          except IOError, e:
    208            # If not retrying, then just pass on the error.
    209            if timeout == 0:
    210              raise e
    211            if e.errno != errno.EACCES:
    212              raise e
    213            # We could not acquire the lock. Try again.
    214            if (time.time() - start_time) >= timeout:
    215              logger.warn('Could not lock %s in %s seconds' % (
    216                  self._filename, timeout))
    217              if self._fh:
    218                self._fh.close()
    219              self._fh = open(self._filename, self._fallback_mode)
    220              return
    221            time.sleep(delay)
    222
    223 -    def unlock_and_close(self):
    224        """Close and unlock the file using the fcntl.lockf primitive."""
    225        if self._locked:
    226          fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN)
    227        self._locked = False
    228        if self._fh:
    229          self._fh.close()
    230  except ImportError:
    231    _FcntlOpener = None
    232
    233
    234  try:
    235    import pywintypes
    236    import win32con
    237    import win32file
    238
    239 -  class _Win32Opener(_Opener):
    240      """Open, lock, and unlock a file using windows primitives."""
    241
    242      # Error #33:
    243      #  'The process cannot access the file because another process'
    244      FILE_IN_USE_ERROR = 33
    245
    246      # Error #158:
    247      #  'The segment is already unlocked.'
    248      FILE_ALREADY_UNLOCKED_ERROR = 158
    249
    250 -    def open_and_lock(self, timeout, delay):
    251        """Open the file and lock it.
    252
    253        Args:
    254          timeout: float, How long to try to lock for.
    255          delay: float, How long to wait between retries
    256
    257        Raises:
    258          AlreadyLockedException: if the lock is already acquired.
    259          IOError: if the open fails.
    260          CredentialsFileSymbolicLinkError if the file is a symbolic link.
    261        """
    262        if self._locked:
    263          raise AlreadyLockedException('File %s is already locked' %
    264                                       self._filename)
    265        start_time = time.time()
    266
    267        validate_file(self._filename)
    268        try:
    269          self._fh = open(self._filename, self._mode)
    270        except IOError, e:
    271          # If we can't access with _mode, try _fallback_mode and don't lock.
    272          if e.errno == errno.EACCES:
    273            self._fh = open(self._filename, self._fallback_mode)
    274            return
    275
    276        # We opened in _mode, try to lock the file.
    277        while True:
    278          try:
    279            hfile = win32file._get_osfhandle(self._fh.fileno())
    280            win32file.LockFileEx(
    281                hfile,
    282                (win32con.LOCKFILE_FAIL_IMMEDIATELY|
    283                 win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000,
    284                pywintypes.OVERLAPPED())
    285            self._locked = True
    286            return
    287          except pywintypes.error, e:
    288            if timeout == 0:
    289              raise e
    290
    291            # If the error is not that the file is already in use, raise.
    292            if e[0] != _Win32Opener.FILE_IN_USE_ERROR:
    293              raise
    294
    295            # We could not acquire the lock. Try again.
    296            if (time.time() - start_time) >= timeout:
    297              logger.warn('Could not lock %s in %s seconds' % (
    298                  self._filename, timeout))
    299              if self._fh:
    300                self._fh.close()
    301              self._fh = open(self._filename, self._fallback_mode)
    302              return
    303            time.sleep(delay)
    304
    305 -    def unlock_and_close(self):
    306        """Close and unlock the file using the win32 primitive."""
    307        if self._locked:
    308          try:
    309            hfile = win32file._get_osfhandle(self._fh.fileno())
    310            win32file.UnlockFileEx(hfile, 0, -0x10000, pywintypes.OVERLAPPED())
    311          except pywintypes.error, e:
    312            if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR:
    313              raise
    314        self._locked = False
    315        if self._fh:
    316          self._fh.close()
    317  except ImportError:
    318    _Win32Opener = None
    319
    320
    321 -class LockedFile(object):
    322    """Represent a file that has exclusive access."""
    323
    324    @util.positional(4)
    325 -  def __init__(self, filename, mode, fallback_mode, use_native_locking=True):
    326      """Construct a LockedFile.
    327
    328      Args:
    329        filename: string, The path of the file to open.
    330        mode: string, The mode to try to open the file with.
    331        fallback_mode: string, The mode to use if locking fails.
    332        use_native_locking: bool, Whether or not fcntl/win32 locking is used.
    333      """
    334      opener = None
    335      if not opener and use_native_locking:
    336        if _Win32Opener:
    337          opener = _Win32Opener(filename, mode, fallback_mode)
    338        if _FcntlOpener:
    339          opener = _FcntlOpener(filename, mode, fallback_mode)
    340
    341      if not opener:
    342        opener = _PosixOpener(filename, mode, fallback_mode)
    343
    344      self._opener = opener
    345
    346 -  def filename(self):
    347      """Return the filename we were constructed with."""
    348      return self._opener._filename
    349
    350 -  def file_handle(self):
    351      """Return the file_handle to the opened file."""
    352      return self._opener.file_handle()
    353
    354 -  def is_locked(self):
    355      """Return whether we successfully locked the file."""
    356      return self._opener.is_locked()
    357
    358 -  def open_and_lock(self, timeout=0, delay=0.05):
    359      """Open the file, trying to lock it.
    360
    361      Args:
    362        timeout: float, The number of seconds to try to acquire the lock.
    363        delay: float, The number of seconds to wait between retry attempts.
    364
    365      Raises:
    366        AlreadyLockedException: if the lock is already acquired.
    367        IOError: if the open fails.
    368      """
    369      self._opener.open_and_lock(timeout, delay)
    370
    371 -  def unlock_and_close(self):
    372      """Unlock and close a file."""
    373      self._opener.unlock_and_close()
    374