Package apiclient :: Module http |
[hide private]
[frames] | no frames]

# Source Code for Module apiclient.http

       1  # Copyright (C) 2012 Google Inc.
       2  #
       3  # Licensed under the Apache License, Version 2.0 (the "License");
       4  # you may not use this file except in compliance with the License.
       5  # You may obtain a copy of the License at
       6  #
       7  #      http://www.apache.org/licenses/LICENSE-2.0
       8  #
       9  # Unless required by applicable law or agreed to in writing, software
      10  # distributed under the License is distributed on an "AS IS" BASIS,
      11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      12  # See the License for the specific language governing permissions and
      13  # limitations under the License.
      14
      15  """Classes to encapsulate a single HTTP request.
      16
      17  The classes implement a command pattern, with every
      18  object supporting an execute() method that does the
      19  actuall HTTP request.
      20  """
      21
      22  __author__ = 'jcgregorio@google.com (Joe Gregorio)'
      23
      24  import StringIO
      25  import base64
      26  import copy
      27  import gzip
      28  import httplib2
      29  import logging
      30  import mimeparse
      31  import mimetypes
      32  import os
      33  import random
      34  import sys
      35  import time
      36  import urllib
      37  import urlparse
      38  import uuid
      39
      40  from email.generator import Generator
      41  from email.mime.multipart import MIMEMultipart
      42  from email.mime.nonmultipart import MIMENonMultipart
      43  from email.parser import FeedParser
      44  from errors import BatchError
      45  from errors import HttpError
      46  from errors import InvalidChunkSizeError
      47  from errors import ResumableUploadError
      48  from errors import UnexpectedBodyError
      49  from errors import UnexpectedMethodError
      50  from model import JsonModel
      51  from oauth2client import util
      52  from oauth2client.anyjson import simplejson
      53
      54
      55  DEFAULT_CHUNK_SIZE = 512*1024
      56
      57  MAX_URI_LENGTH = 2048
      58
      59
      60 -class MediaUploadProgress(object):
      61    """Status of a resumable upload."""
      62
      63 -  def __init__(self, resumable_progress, total_size):
      64      """Constructor.
      65
      66      Args:
      67        resumable_progress: int, bytes sent so far.
      68        total_size: int, total bytes in complete upload, or None if the total
      69          upload size isn't known ahead of time.
      70      """
      71      self.resumable_progress = resumable_progress
      72      self.total_size = total_size
      73
      74 -  def progress(self):
      75      """Percent of upload completed, as a float.
      76
      77      Returns:
      78        the percentage complete as a float, returning 0.0 if the total size of
      79        the upload is unknown.
      80      """
      81      if self.total_size is not None:
      82        return float(self.resumable_progress) / float(self.total_size)
      83      else:
      84        return 0.0
      85
      86
      87 -class MediaDownloadProgress(object):
      88    """Status of a resumable download."""
      89
      90 -  def __init__(self, resumable_progress, total_size):
      91      """Constructor.
      92
      93      Args:
      94        resumable_progress: int, bytes received so far.
      95        total_size: int, total bytes in complete download.
      96      """
      97      self.resumable_progress = resumable_progress
      98      self.total_size = total_size
      99
     100 -  def progress(self):
     101      """Percent of download completed, as a float.
     102
     103      Returns:
     104        the percentage complete as a float, returning 0.0 if the total size of
     105        the download is unknown.
     106      """
     107      if self.total_size is not None:
     108        return float(self.resumable_progress) / float(self.total_size)
     109      else:
     110        return 0.0
     111
     112
     113 -class MediaUpload(object):
     114    """Describes a media object to upload.
     115
     116    Base class that defines the interface of MediaUpload subclasses.
     117
     118    Note that subclasses of MediaUpload may allow you to control the chunksize
     119    when uploading a media object. It is important to keep the size of the chunk
     120    as large as possible to keep the upload efficient. Other factors may influence
     121    the size of the chunk you use, particularly if you are working in an
     122    environment where individual HTTP requests may have a hardcoded time limit,
     123    such as under certain classes of requests under Google App Engine.
     124
     125    Streams are io.Base compatible objects that support seek(). Some MediaUpload
     126    subclasses support using streams directly to upload data. Support for
     127    streaming may be indicated by a MediaUpload sub-class and if appropriate for a
     128    platform that stream will be used for uploading the media object. The support
     129    for streaming is indicated by has_stream() returning True. The stream() method
     130    should return an io.Base object that supports seek(). On platforms where the
     131    underlying httplib module supports streaming, for example Python 2.6 and
     132    later, the stream will be passed into the http library which will result in
     133    less memory being used and possibly faster uploads.
     134
     135    If you need to upload media that can't be uploaded using any of the existing
     136    MediaUpload sub-class then you can sub-class MediaUpload for your particular
     137    needs.
     138    """
     139
     140 -  def chunksize(self):
     141      """Chunk size for resumable uploads.
     142
     143      Returns:
     144        Chunk size in bytes.
     145      """
     146      raise NotImplementedError()
     147
     148 -  def mimetype(self):
     149      """Mime type of the body.
     150
     151      Returns:
     152        Mime type.
     153      """
     154      return 'application/octet-stream'
     155
     156 -  def size(self):
     157      """Size of upload.
     158
     159      Returns:
     160        Size of the body, or None of the size is unknown.
     161      """
     162      return None
     163
     164 -  def resumable(self):
     165      """Whether this upload is resumable.
     166
     167      Returns:
     168        True if resumable upload or False.
     169      """
     170      return False
     171
     172 -  def getbytes(self, begin, end):
     173      """Get bytes from the media.
     174
     175      Args:
     176        begin: int, offset from beginning of file.
     177        length: int, number of bytes to read, starting at begin.
     178
     179      Returns:
     180        A string of bytes read. May be shorter than length if EOF was reached
     181        first.
     182      """
     183      raise NotImplementedError()
     184
     185 -  def has_stream(self):
     186      """Does the underlying upload support a streaming interface.
     187
     188      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     189      seekable() returns True.
     190
     191      Returns:
     192        True if the call to stream() will return an instance of a seekable io.Base
     193        subclass.
     194      """
     195      return False
     196
     197 -  def stream(self):
     198      """A stream interface to the data being uploaded.
     199
     200      Returns:
     201        The returned value is an io.IOBase subclass that supports seek, i.e.
     202        seekable() returns True.
     203      """
     204      raise NotImplementedError()
     205
     206    @util.positional(1)
     207 -  def _to_json(self, strip=None):
     208      """Utility function for creating a JSON representation of a MediaUpload.
     209
     210      Args:
     211        strip: array, An array of names of members to not include in the JSON.
     212
     213      Returns:
     214         string, a JSON representation of this instance, suitable to pass to
     215         from_json().
     216      """
     217      t = type(self)
     218      d = copy.copy(self.__dict__)
     219      if strip is not None:
     220        for member in strip:
     221          del d[member]
     222      d['_class'] = t.__name__
     223      d['_module'] = t.__module__
     224      return simplejson.dumps(d)
     225
     226 -  def to_json(self):
     227      """Create a JSON representation of an instance of MediaUpload.
     228
     229      Returns:
     230         string, a JSON representation of this instance, suitable to pass to
     231         from_json().
     232      """
     233      return self._to_json()
     234
     235    @classmethod
     236 -  def new_from_json(cls, s):
     237      """Utility class method to instantiate a MediaUpload subclass from a JSON
     238      representation produced by to_json().
     239
     240      Args:
     241        s: string, JSON from to_json().
     242
     243      Returns:
     244        An instance of the subclass of MediaUpload that was serialized with
     245        to_json().
     246      """
     247      data = simplejson.loads(s)
     248      # Find and call the right classmethod from_json() to restore the object.
     249      module = data['_module']
     250      m = __import__(module, fromlist=module.split('.')[:-1])
     251      kls = getattr(m, data['_class'])
     252      from_json = getattr(kls, 'from_json')
     253      return from_json(s)
     254
     255
     256 -class MediaIoBaseUpload(MediaUpload):
     257    """A MediaUpload for a io.Base objects.
     258
     259    Note that the Python file object is compatible with io.Base and can be used
     260    with this class also.
     261
     262      fh = io.BytesIO('...Some data to upload...')
     263      media = MediaIoBaseUpload(fh, mimetype='image/png',
     264        chunksize=1024*1024, resumable=True)
     265      farm.animals().insert(
     266          id='cow',
     267          name='cow.png',
     268          media_body=media).execute()
     269
     270    Depending on the platform you are working on, you may pass -1 as the
     271    chunksize, which indicates that the entire file should be uploaded in a single
     272    request. If the underlying platform supports streams, such as Python 2.6 or
     273    later, then this can be very efficient as it avoids multiple connections, and
     274    also avoids loading the entire file into memory before sending it. Note that
     275    Google App Engine has a 5MB limit on request size, so you should never set
     276    your chunksize larger than 5MB, or to -1.
     277    """
     278
     279    @util.positional(3)
     280 -  def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE,
     281        resumable=False):
     282      """Constructor.
     283
     284      Args:
     285        fd: io.Base or file object, The source of the bytes to upload. MUST be
     286          opened in blocking mode, do not use streams opened in non-blocking mode.
     287          The given stream must be seekable, that is, it must be able to call
     288          seek() on fd.
     289        mimetype: string, Mime-type of the file.
     290        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     291          used if resumable=True. Pass in a value of -1 if the file is to be
     292          uploaded as a single chunk. Note that Google App Engine has a 5MB limit
     293          on request size, so you should never set your chunksize larger than 5MB,
     294          or to -1.
     295        resumable: bool, True if this is a resumable upload. False means upload
     296          in a single request.
     297      """
     298      super(MediaIoBaseUpload, self).__init__()
     299      self._fd = fd
     300      self._mimetype = mimetype
     301      if not (chunksize == -1 or chunksize > 0):
     302        raise InvalidChunkSizeError()
     303      self._chunksize = chunksize
     304      self._resumable = resumable
     305
     306      self._fd.seek(0, os.SEEK_END)
     307      self._size = self._fd.tell()
     308
     309 -  def chunksize(self):
     310      """Chunk size for resumable uploads.
     311
     312      Returns:
     313        Chunk size in bytes.
     314      """
     315      return self._chunksize
     316
     317 -  def mimetype(self):
     318      """Mime type of the body.
     319
     320      Returns:
     321        Mime type.
     322      """
     323      return self._mimetype
     324
     325 -  def size(self):
     326      """Size of upload.
     327
     328      Returns:
     329        Size of the body, or None of the size is unknown.
     330      """
     331      return self._size
     332
     333 -  def resumable(self):
     334      """Whether this upload is resumable.
     335
     336      Returns:
     337        True if resumable upload or False.
     338      """
     339      return self._resumable
     340
     341 -  def getbytes(self, begin, length):
     342      """Get bytes from the media.
     343
     344      Args:
     345        begin: int, offset from beginning of file.
     346        length: int, number of bytes to read, starting at begin.
     347
     348      Returns:
     349        A string of bytes read. May be shorted than length if EOF was reached
     350        first.
     351      """
     352      self._fd.seek(begin)
     353      return self._fd.read(length)
     354
     355 -  def has_stream(self):
     356      """Does the underlying upload support a streaming interface.
     357
     358      Streaming means it is an io.IOBase subclass that supports seek, i.e.
     359      seekable() returns True.
     360
     361      Returns:
     362        True if the call to stream() will return an instance of a seekable io.Base
     363        subclass.
     364      """
     365      return True
     366
     367 -  def stream(self):
     368      """A stream interface to the data being uploaded.
     369
     370      Returns:
     371        The returned value is an io.IOBase subclass that supports seek, i.e.
     372        seekable() returns True.
     373      """
     374      return self._fd
     375
     376 -  def to_json(self):
     377      """This upload type is not serializable."""
     378      raise NotImplementedError('MediaIoBaseUpload is not serializable.')
     379
     380
     381 -class MediaFileUpload(MediaIoBaseUpload):
     382    """A MediaUpload for a file.
     383
     384    Construct a MediaFileUpload and pass as the media_body parameter of the
     385    method. For example, if we had a service that allowed uploading images:
     386
     387
     388      media = MediaFileUpload('cow.png', mimetype='image/png',
     389        chunksize=1024*1024, resumable=True)
     390      farm.animals().insert(
     391          id='cow',
     392          name='cow.png',
     393          media_body=media).execute()
     394
     395    Depending on the platform you are working on, you may pass -1 as the
     396    chunksize, which indicates that the entire file should be uploaded in a single
     397    request. If the underlying platform supports streams, such as Python 2.6 or
     398    later, then this can be very efficient as it avoids multiple connections, and
     399    also avoids loading the entire file into memory before sending it. Note that
     400    Google App Engine has a 5MB limit on request size, so you should never set
     401    your chunksize larger than 5MB, or to -1.
     402    """
     403
     404    @util.positional(2)
     405 -  def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE,
     406                 resumable=False):
     407      """Constructor.
     408
     409      Args:
     410        filename: string, Name of the file.
     411        mimetype: string, Mime-type of the file. If None then a mime-type will be
     412          guessed from the file extension.
     413        chunksize: int, File will be uploaded in chunks of this many bytes. Only
     414          used if resumable=True. Pass in a value of -1 if the file is to be
     415          uploaded in a single chunk. Note that Google App Engine has a 5MB limit
     416          on request size, so you should never set your chunksize larger than 5MB,
     417          or to -1.
     418        resumable: bool, True if this is a resumable upload. False means upload
     419          in a single request.
     420      """
     421      self._filename = filename
     422      fd = open(self._filename, 'rb')
     423      if mimetype is None:
     424        (mimetype, encoding) = mimetypes.guess_type(filename)
     425      super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     426                                            resumable=resumable)
     427
     428 -  def to_json(self):
     429      """Creating a JSON representation of an instance of MediaFileUpload.
     430
     431      Returns:
     432         string, a JSON representation of this instance, suitable to pass to
     433         from_json().
     434      """
     435      return self._to_json(strip=['_fd'])
     436
     437    @staticmethod
     438 -  def from_json(s):
     439      d = simplejson.loads(s)
     440      return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'],
     441                             chunksize=d['_chunksize'], resumable=d['_resumable'])
     442
     443
     444 -class MediaInMemoryUpload(MediaIoBaseUpload):
     445    """MediaUpload for a chunk of bytes.
     446
     447    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     448    the stream.
     449    """
     450
     451    @util.positional(2)
     452 -  def __init__(self, body, mimetype='application/octet-stream',
     453                 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):
     454      """Create a new MediaInMemoryUpload.
     455
     456    DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for
     457    the stream.
     458
     459    Args:
     460      body: string, Bytes of body content.
     461      mimetype: string, Mime-type of the file or default of
     462        'application/octet-stream'.
     463      chunksize: int, File will be uploaded in chunks of this many bytes. Only
     464        used if resumable=True.
     465      resumable: bool, True if this is a resumable upload. False means upload
     466        in a single request.
     467      """
     468      fd = StringIO.StringIO(body)
     469      super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize,
     470                                                resumable=resumable)
     471
     472
     473 -class MediaIoBaseDownload(object):
     474    """"Download media resources.
     475
     476    Note that the Python file object is compatible with io.Base and can be used
     477    with this class also.
     478
     479
     480    Example:
     481      request = farms.animals().get_media(id='cow')
     482      fh = io.FileIO('cow.png', mode='wb')
     483      downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024)
     484
     485      done = False
     486      while done is False:
     487        status, done = downloader.next_chunk()
     488        if status:
     489          print "Download %d%%." % int(status.progress() * 100)
     490      print "Download Complete!"
     491    """
     492
     493    @util.positional(3)
     494 -  def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):
     495      """Constructor.
     496
     497      Args:
     498        fd: io.Base or file object, The stream in which to write the downloaded
     499          bytes.
     500        request: apiclient.http.HttpRequest, the media request to perform in
     501          chunks.
     502        chunksize: int, File will be downloaded in chunks of this many bytes.
     503      """
     504      self._fd = fd
     505      self._request = request
     506      self._uri = request.uri
     507      self._chunksize = chunksize
     508      self._progress = 0
     509      self._total_size = None
     510      self._done = False
     511
     512      # Stubs for testing.
     513      self._sleep = time.sleep
     514      self._rand = random.random
     515
     516    @util.positional(1)
     517 -  def next_chunk(self, num_retries=0):
     518      """Get the next chunk of the download.
     519
     520      Args:
     521        num_retries: Integer, number of times to retry 500's with randomized
     522              exponential backoff. If all retries fail, the raised HttpError
     523              represents the last request. If zero (default), we attempt the
     524              request only once.
     525
     526      Returns:
     527        (status, done): (MediaDownloadStatus, boolean)
     528           The value of 'done' will be True when the media has been fully
     529           downloaded.
     530
     531      Raises:
     532        apiclient.errors.HttpError if the response was not a 2xx.
     533        httplib2.HttpLib2Error if a transport error has occured.
     534      """
     535      headers = {
     536          'range': 'bytes=%d-%d' % (
     537              self._progress, self._progress + self._chunksize)
     538          }
     539      http = self._request.http
     540
     541      for retry_num in xrange(num_retries + 1):
     542        if retry_num > 0:
     543          self._sleep(self._rand() * 2**retry_num)
     544          logging.warning(
     545              'Retry #%d for media download: GET %s, following status: %d'
     546              % (retry_num, self._uri, resp.status))
     547
     548        resp, content = http.request(self._uri, headers=headers)
     549        if resp.status < 500:
     550          break
     551
     552      if resp.status in [200, 206]:
     553        if 'content-location' in resp and resp['content-location'] != self._uri:
     554          self._uri = resp['content-location']
     555        self._progress += len(content)
     556        self._fd.write(content)
     557
     558        if 'content-range' in resp:
     559          content_range = resp['content-range']
     560          length = content_range.rsplit('/', 1)[1]
     561          self._total_size = int(length)
     562
     563        if self._progress == self._total_size:
     564          self._done = True
     565        return MediaDownloadProgress(self._progress, self._total_size), self._done
     566      else:
     567        raise HttpError(resp, content, uri=self._uri)
     568
     569
     570 -class _StreamSlice(object):
     571    """Truncated stream.
     572
     573    Takes a stream and presents a stream that is a slice of the original stream.
     574    This is used when uploading media in chunks. In later versions of Python a
     575    stream can be passed to httplib in place of the string of data to send. The
     576    problem is that httplib just blindly reads to the end of the stream. This
     577    wrapper presents a virtual stream that only reads to the end of the chunk.
     578    """
     579
     580 -  def __init__(self, stream, begin, chunksize):
     581      """Constructor.
     582
     583      Args:
     584        stream: (io.Base, file object), the stream to wrap.
     585        begin: int, the seek position the chunk begins at.
     586        chunksize: int, the size of the chunk.
     587      """
     588      self._stream = stream
     589      self._begin = begin
     590      self._chunksize = chunksize
     591      self._stream.seek(begin)
     592
     593 -  def read(self, n=-1):
     594      """Read n bytes.
     595
     596      Args:
     597        n, int, the number of bytes to read.
     598
     599      Returns:
     600        A string of length 'n', or less if EOF is reached.
     601      """
     602      # The data left available to read sits in [cur, end)
     603      cur = self._stream.tell()
     604      end = self._begin + self._chunksize
     605      if n == -1 or cur + n > end:
     606        n = end - cur
     607      return self._stream.read(n)
     608
     609
     610 -class HttpRequest(object):
     611    """Encapsulates a single HTTP request."""
     612
     613    @util.positional(4)
     614 -  def __init__(self, http, postproc, uri,
     615                 method='GET',
     616                 body=None,
     617                 headers=None,
     618                 methodId=None,
     619                 resumable=None):
     620      """Constructor for an HttpRequest.
     621
     622      Args:
     623        http: httplib2.Http, the transport object to use to make a request
     624        postproc: callable, called on the HTTP response and content to transform
     625                  it into a data object before returning, or raising an exception
     626                  on an error.
     627        uri: string, the absolute URI to send the request to
     628        method: string, the HTTP method to use
     629        body: string, the request body of the HTTP request,
     630        headers: dict, the HTTP request headers
     631        methodId: string, a unique identifier for the API method being called.
     632        resumable: MediaUpload, None if this is not a resumbale request.
     633      """
     634      self.uri = uri
     635      self.method = method
     636      self.body = body
     637      self.headers = headers or {}
     638      self.methodId = methodId
     639      self.http = http
     640      self.postproc = postproc
     641      self.resumable = resumable
     642      self.response_callbacks = []
     643      self._in_error_state = False
     644
     645      # Pull the multipart boundary out of the content-type header.
     646      major, minor, params = mimeparse.parse_mime_type(
     647          headers.get('content-type', 'application/json'))
     648
     649      # The size of the non-media part of the request.
     650      self.body_size = len(self.body or '')
     651
     652      # The resumable URI to send chunks to.
     653      self.resumable_uri = None
     654
     655      # The bytes that have been uploaded.
     656      self.resumable_progress = 0
     657
     658      # Stubs for testing.
     659      self._rand = random.random
     660      self._sleep = time.sleep
     661
     662    @util.positional(1)
     663 -  def execute(self, http=None, num_retries=0):
     664      """Execute the request.
     665
     666      Args:
     667        http: httplib2.Http, an http object to be used in place of the
     668              one the HttpRequest request object was constructed with.
     669        num_retries: Integer, number of times to retry 500's with randomized
     670              exponential backoff. If all retries fail, the raised HttpError
     671              represents the last request. If zero (default), we attempt the
     672              request only once.
     673
     674      Returns:
     675        A deserialized object model of the response body as determined
     676        by the postproc.
     677
     678      Raises:
     679        apiclient.errors.HttpError if the response was not a 2xx.
     680        httplib2.HttpLib2Error if a transport error has occured.
     681      """
     682      if http is None:
     683        http = self.http
     684
     685      if self.resumable:
     686        body = None
     687        while body is None:
     688          _, body = self.next_chunk(http=http, num_retries=num_retries)
     689        return body
     690
     691      # Non-resumable case.
     692
     693      if 'content-length' not in self.headers:
     694        self.headers['content-length'] = str(self.body_size)
     695      # If the request URI is too long then turn it into a POST request.
     696      if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET':
     697        self.method = 'POST'
     698        self.headers['x-http-method-override'] = 'GET'
     699        self.headers['content-type'] = 'application/x-www-form-urlencoded'
     700        parsed = urlparse.urlparse(self.uri)
     701        self.uri = urlparse.urlunparse(
     702            (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None,
     703             None)
     704            )
     705        self.body = parsed.query
     706        self.headers['content-length'] = str(len(self.body))
     707
     708      # Handle retries for server-side errors.
     709      for retry_num in xrange(num_retries + 1):
     710        if retry_num > 0:
     711          self._sleep(self._rand() * 2**retry_num)
     712          logging.warning('Retry #%d for request: %s %s, following status: %d'
     713                          % (retry_num, self.method, self.uri, resp.status))
     714
     715        resp, content = http.request(str(self.uri), method=str(self.method),
     716                                     body=self.body, headers=self.headers)
     717        if resp.status < 500:
     718          break
     719
     720      for callback in self.response_callbacks:
     721        callback(resp)
     722      if resp.status >= 300:
     723        raise HttpError(resp, content, uri=self.uri)
     724      return self.postproc(resp, content)
     725
     726    @util.positional(2)
     727 -  def add_response_callback(self, cb):
     728      """add_response_headers_callback
     729
     730      Args:
     731        cb: Callback to be called on receiving the response headers, of signature:
     732
     733        def cb(resp):
     734          # Where resp is an instance of httplib2.Response
     735      """
     736      self.response_callbacks.append(cb)
     737
     738    @util.positional(1)
     739 -  def next_chunk(self, http=None, num_retries=0):
     740      """Execute the next step of a resumable upload.
     741
     742      Can only be used if the method being executed supports media uploads and
     743      the MediaUpload object passed in was flagged as using resumable upload.
     744
     745      Example:
     746
     747        media = MediaFileUpload('cow.png', mimetype='image/png',
     748                                chunksize=1000, resumable=True)
     749        request = farm.animals().insert(
     750            id='cow',
     751            name='cow.png',
     752            media_body=media)
     753
     754        response = None
     755        while response is None:
     756          status, response = request.next_chunk()
     757          if status:
     758            print "Upload %d%% complete." % int(status.progress() * 100)
     759
     760
     761      Args:
     762        http: httplib2.Http, an http object to be used in place of the
     763              one the HttpRequest request object was constructed with.
     764        num_retries: Integer, number of times to retry 500's with randomized
     765              exponential backoff. If all retries fail, the raised HttpError
     766              represents the last request. If zero (default), we attempt the
     767              request only once.
     768
     769      Returns:
     770        (status, body): (ResumableMediaStatus, object)
     771           The body will be None until the resumable media is fully uploaded.
     772
     773      Raises:
     774        apiclient.errors.HttpError if the response was not a 2xx.
     775        httplib2.HttpLib2Error if a transport error has occured.
     776      """
     777      if http is None:
     778        http = self.http
     779
     780      if self.resumable.size() is None:
     781        size = '*'
     782      else:
     783        size = str(self.resumable.size())
     784
     785      if self.resumable_uri is None:
     786        start_headers = copy.copy(self.headers)
     787        start_headers['X-Upload-Content-Type'] = self.resumable.mimetype()
     788        if size != '*':
     789          start_headers['X-Upload-Content-Length'] = size
     790        start_headers['content-length'] = str(self.body_size)
     791
     792        for retry_num in xrange(num_retries + 1):
     793          if retry_num > 0:
     794            self._sleep(self._rand() * 2**retry_num)
     795            logging.warning(
     796                'Retry #%d for resumable URI request: %s %s, following status: %d'
     797                % (retry_num, self.method, self.uri, resp.status))
     798
     799          resp, content = http.request(self.uri, method=self.method,
     800                                       body=self.body,
     801                                       headers=start_headers)
     802          if resp.status < 500:
     803            break
     804
     805        if resp.status == 200 and 'location' in resp:
     806          self.resumable_uri = resp['location']
     807        else:
     808          raise ResumableUploadError(resp, content)
     809      elif self._in_error_state:
     810        # If we are in an error state then query the server for current state of
     811        # the upload by sending an empty PUT and reading the 'range' header in
     812        # the response.
     813        headers = {
     814            'Content-Range': 'bytes */%s' % size,
     815            'content-length': '0'
     816            }
     817        resp, content = http.request(self.resumable_uri, 'PUT',
     818                                     headers=headers)
     819        status, body = self._process_response(resp, content)
     820        if body:
     821          # The upload was complete.
     822          return (status, body)
     823
     824      # The httplib.request method can take streams for the body parameter, but
     825      # only in Python 2.6 or later. If a stream is available under those
     826      # conditions then use it as the body argument.
     827      if self.resumable.has_stream() and sys.version_info[1] >= 6:
     828        data = self.resumable.stream()
     829        if self.resumable.chunksize() == -1:
     830          data.seek(self.resumable_progress)
     831          chunk_end = self.resumable.size() - self.resumable_progress - 1
     832        else:
     833          # Doing chunking with a stream, so wrap a slice of the stream.
     834          data = _StreamSlice(data, self.resumable_progress,
     835                              self.resumable.chunksize())
     836          chunk_end = min(
     837              self.resumable_progress + self.resumable.chunksize() - 1,
     838              self.resumable.size() - 1)
     839      else:
     840        data = self.resumable.getbytes(
     841            self.resumable_progress, self.resumable.chunksize())
     842
     843        # A short read implies that we are at EOF, so finish the upload.
     844        if len(data) < self.resumable.chunksize():
     845          size = str(self.resumable_progress + len(data))
     846
     847        chunk_end = self.resumable_progress + len(data) - 1
     848
     849      headers = {
     850          'Content-Range': 'bytes %d-%d/%s' % (
     851              self.resumable_progress, chunk_end, size),
     852          # Must set the content-length header here because httplib can't
     853          # calculate the size when working with _StreamSlice.
     854          'Content-Length': str(chunk_end - self.resumable_progress + 1)
     855          }
     856
     857      for retry_num in xrange(num_retries + 1):
     858        if retry_num > 0:
     859          self._sleep(self._rand() * 2**retry_num)
     860          logging.warning(
     861              'Retry #%d for media upload: %s %s, following status: %d'
     862              % (retry_num, self.method, self.uri, resp.status))
     863
     864        try:
     865          resp, content = http.request(self.resumable_uri, method='PUT',
     866                                       body=data,
     867                                       headers=headers)
     868        except:
     869          self._in_error_state = True
     870          raise
     871        if resp.status < 500:
     872          break
     873
     874      return self._process_response(resp, content)
     875
     876 -  def _process_response(self, resp, content):
     877      """Process the response from a single chunk upload.
     878
     879      Args:
     880        resp: httplib2.Response, the response object.
     881        content: string, the content of the response.
     882
     883      Returns:
     884        (status, body): (ResumableMediaStatus, object)
     885           The body will be None until the resumable media is fully uploaded.
     886
     887      Raises:
     888        apiclient.errors.HttpError if the response was not a 2xx or a 308.
     889      """
     890      if resp.status in [200, 201]:
     891        self._in_error_state = False
     892        return None, self.postproc(resp, content)
     893      elif resp.status == 308:
     894        self._in_error_state = False
     895        # A "308 Resume Incomplete" indicates we are not done.
     896        self.resumable_progress = int(resp['range'].split('-')[1]) + 1
     897        if 'location' in resp:
     898          self.resumable_uri = resp['location']
     899      else:
     900        self._in_error_state = True
     901        raise HttpError(resp, content, uri=self.uri)
     902
     903      return (MediaUploadProgress(self.resumable_progress, self.resumable.size()),
     904              None)
     905
     906 -  def to_json(self):
     907      """Returns a JSON representation of the HttpRequest."""
     908      d = copy.copy(self.__dict__)
     909      if d['resumable'] is not None:
     910        d['resumable'] = self.resumable.to_json()
     911      del d['http']
     912      del d['postproc']
     913      del d['_sleep']
     914      del d['_rand']
     915
     916      return simplejson.dumps(d)
     917
     918    @staticmethod
     919 -  def from_json(s, http, postproc):
     920      """Returns an HttpRequest populated with info from a JSON object."""
     921      d = simplejson.loads(s)
     922      if d['resumable'] is not None:
     923        d['resumable'] = MediaUpload.new_from_json(d['resumable'])
     924      return HttpRequest(
     925          http,
     926          postproc,
     927          uri=d['uri'],
     928          method=d['method'],
     929          body=d['body'],
     930          headers=d['headers'],
     931          methodId=d['methodId'],
     932          resumable=d['resumable'])
     933
     934
     935 -class BatchHttpRequest(object):
     936    """Batches multiple HttpRequest objects into a single HTTP request.
     937
     938    Example:
     939      from apiclient.http import BatchHttpRequest
     940
     941      def list_animals(request_id, response, exception):
     942        \"\"\"Do something with the animals list response.\"\"\"
     943        if exception is not None:
     944          # Do something with the exception.
     945          pass
     946        else:
     947          # Do something with the response.
     948          pass
     949
     950      def list_farmers(request_id, response, exception):
     951        \"\"\"Do something with the farmers list response.\"\"\"
     952        if exception is not None:
     953          # Do something with the exception.
     954          pass
     955        else:
     956          # Do something with the response.
     957          pass
     958
     959      service = build('farm', 'v2')
     960
     961      batch = BatchHttpRequest()
     962
     963      batch.add(service.animals().list(), list_animals)
     964      batch.add(service.farmers().list(), list_farmers)
     965      batch.execute(http=http)
     966    """
     967
     968    @util.positional(1)
     969 -  def __init__(self, callback=None, batch_uri=None):
     970      """Constructor for a BatchHttpRequest.
     971
     972      Args:
     973        callback: callable, A callback to be called for each response, of the
     974          form callback(id, response, exception). The first parameter is the
     975          request id, and the second is the deserialized response object. The
     976          third is an apiclient.errors.HttpError exception object if an HTTP error
     977          occurred while processing the request, or None if no error occurred.
     978        batch_uri: string, URI to send batch requests to.
     979      """
     980      if batch_uri is None:
     981        batch_uri = 'https://www.googleapis.com/batch'
     982      self._batch_uri = batch_uri
     983
     984      # Global callback to be called for each individual response in the batch.
     985      self._callback = callback
     986
     987      # A map from id to request.
     988      self._requests = {}
     989
     990      # A map from id to callback.
     991      self._callbacks = {}
     992
     993      # List of request ids, in the order in which they were added.
     994      self._order = []
     995
     996      # The last auto generated id.
     997      self._last_auto_id = 0
     998
     999      # Unique ID on which to base the Content-ID headers.
    1000      self._base_id = None
    1001
    1002      # A map from request id to (httplib2.Response, content) response pairs
    1003      self._responses = {}
    1004
    1005      # A map of id(Credentials) that have been refreshed.
    1006      self._refreshed_credentials = {}
    1007
    1008 -  def _refresh_and_apply_credentials(self, request, http):
    1009      """Refresh the credentials and apply to the request.
    1010
    1011      Args:
    1012        request: HttpRequest, the request.
    1013        http: httplib2.Http, the global http object for the batch.
    1014      """
    1015      # For the credentials to refresh, but only once per refresh_token
    1016      # If there is no http per the request then refresh the http passed in
    1017      # via execute()
    1018      creds = None
    1019      if request.http is not None and hasattr(request.http.request,
    1020          'credentials'):
    1021        creds = request.http.request.credentials
    1022      elif http is not None and hasattr(http.request, 'credentials'):
    1023        creds = http.request.credentials
    1024      if creds is not None:
    1025        if id(creds) not in self._refreshed_credentials:
    1026          creds.refresh(http)
    1027          self._refreshed_credentials[id(creds)] = 1
    1028
    1029      # Only apply the credentials if we are using the http object passed in,
    1030      # otherwise apply() will get called during _serialize_request().
    1031      if request.http is None or not hasattr(request.http.request,
    1032          'credentials'):
    1033        creds.apply(request.headers)
    1034
    1035 -  def _id_to_header(self, id_):
    1036      """Convert an id to a Content-ID header value.
    1037
    1038      Args:
    1039        id_: string, identifier of individual request.
    1040
    1041      Returns:
    1042        A Content-ID header with the id_ encoded into it. A UUID is prepended to
    1043        the value because Content-ID headers are supposed to be universally
    1044        unique.
    1045      """
    1046      if self._base_id is None:
    1047        self._base_id = uuid.uuid4()
    1048
    1049      return '<%s+%s>' % (self._base_id, urllib.quote(id_))
    1050
    1051 -  def _header_to_id(self, header):
    1052      """Convert a Content-ID header value to an id.
    1053
    1054      Presumes the Content-ID header conforms to the format that _id_to_header()
    1055      returns.
    1056
    1057      Args:
    1058        header: string, Content-ID header value.
    1059
    1060      Returns:
    1061        The extracted id value.
    1062
    1063      Raises:
    1064        BatchError if the header is not in the expected format.
    1065      """
    1066      if header[0] != '<' or header[-1] != '>':
    1067        raise BatchError("Invalid value for Content-ID: %s" % header)
    1068      if '+' not in header:
    1069        raise BatchError("Invalid value for Content-ID: %s" % header)
    1070      base, id_ = header[1:-1].rsplit('+', 1)
    1071
    1072      return urllib.unquote(id_)
    1073
    1074 -  def _serialize_request(self, request):
    1075      """Convert an HttpRequest object into a string.
    1076
    1077      Args:
    1078        request: HttpRequest, the request to serialize.
    1079
    1080      Returns:
    1081        The request as a string in application/http format.
    1082      """
    1083      # Construct status line
    1084      parsed = urlparse.urlparse(request.uri)
    1085      request_line = urlparse.urlunparse(
    1086          (None, None, parsed.path, parsed.params, parsed.query, None)
    1087          )
    1088      status_line = request.method + ' ' + request_line + ' HTTP/1.1\n'
    1089      major, minor = request.headers.get('content-type', 'application/json').split('/')
    1090      msg = MIMENonMultipart(major, minor)
    1091      headers = request.headers.copy()
    1092
    1093      if request.http is not None and hasattr(request.http.request,
    1094          'credentials'):
    1095        request.http.request.credentials.apply(headers)
    1096
    1097      # MIMENonMultipart adds its own Content-Type header.
    1098      if 'content-type' in headers:
    1099        del headers['content-type']
    1100
    1101      for key, value in headers.iteritems():
    1102        msg[key] = value
    1103      msg['Host'] = parsed.netloc
    1104      msg.set_unixfrom(None)
    1105
    1106      if request.body is not None:
    1107        msg.set_payload(request.body)
    1108        msg['content-length'] = str(len(request.body))
    1109
    1110      # Serialize the mime message.
    1111      fp = StringIO.StringIO()
    1112      # maxheaderlen=0 means don't line wrap headers.
    1113      g = Generator(fp, maxheaderlen=0)
    1114      g.flatten(msg, unixfrom=False)
    1115      body = fp.getvalue()
    1116
    1117      # Strip off the \n\n that the MIME lib tacks onto the end of the payload.
    1118      if request.body is None:
    1119        body = body[:-2]
    1120
    1121      return status_line.encode('utf-8') + body
    1122
    1123 -  def _deserialize_response(self, payload):
    1124      """Convert string into httplib2 response and content.
    1125
    1126      Args:
    1127        payload: string, headers and body as a string.
    1128
    1129      Returns:
    1130        A pair (resp, content), such as would be returned from httplib2.request.
    1131      """
    1132      # Strip off the status line
    1133      status_line, payload = payload.split('\n', 1)
    1134      protocol, status, reason = status_line.split(' ', 2)
    1135
    1136      # Parse the rest of the response
    1137      parser = FeedParser()
    1138      parser.feed(payload)
    1139      msg = parser.close()
    1140      msg['status'] = status
    1141
    1142      # Create httplib2.Response from the parsed headers.
    1143      resp = httplib2.Response(msg)
    1144      resp.reason = reason
    1145      resp.version = int(protocol.split('/', 1)[1].replace('.', ''))
    1146
    1147      content = payload.split('\r\n\r\n', 1)[1]
    1148
    1149      return resp, content
    1150
    1151 -  def _new_id(self):
    1152      """Create a new id.
    1153
    1154      Auto incrementing number that avoids conflicts with ids already used.
    1155
    1156      Returns:
    1157         string, a new unique id.
    1158      """
    1159      self._last_auto_id += 1
    1160      while str(self._last_auto_id) in self._requests:
    1161        self._last_auto_id += 1
    1162      return str(self._last_auto_id)
    1163
    1164    @util.positional(2)
    1165 -  def add(self, request, callback=None, request_id=None):
    1166      """Add a new request.
    1167
    1168      Every callback added will be paired with a unique id, the request_id. That
    1169      unique id will be passed back to the callback when the response comes back
    1170      from the server. The default behavior is to have the library generate it's
    1171      own unique id. If the caller passes in a request_id then they must ensure
    1172      uniqueness for each request_id, and if they are not an exception is
    1173      raised. Callers should either supply all request_ids or nevery supply a
    1174      request id, to avoid such an error.
    1175
    1176      Args:
    1177        request: HttpRequest, Request to add to the batch.
    1178        callback: callable, A callback to be called for this response, of the
    1179          form callback(id, response, exception). The first parameter is the
    1180          request id, and the second is the deserialized response object. The
    1181          third is an apiclient.errors.HttpError exception object if an HTTP error
    1182          occurred while processing the request, or None if no errors occurred.
    1183        request_id: string, A unique id for the request. The id will be passed to
    1184          the callback with the response.
    1185
    1186      Returns:
    1187        None
    1188
    1189      Raises:
    1190        BatchError if a media request is added to a batch.
    1191        KeyError is the request_id is not unique.
    1192      """
    1193      if request_id is None:
    1194        request_id = self._new_id()
    1195      if request.resumable is not None:
    1196        raise BatchError("Media requests cannot be used in a batch request.")
    1197      if request_id in self._requests:
    1198        raise KeyError("A request with this ID already exists: %s" % request_id)
    1199      self._requests[request_id] = request
    1200      self._callbacks[request_id] = callback
    1201      self._order.append(request_id)
    1202
    1203 -  def _execute(self, http, order, requests):
    1204      """Serialize batch request, send to server, process response.
    1205
    1206      Args:
    1207        http: httplib2.Http, an http object to be used to make the request with.
    1208        order: list, list of request ids in the order they were added to the
    1209          batch.
    1210        request: list, list of request objects to send.
    1211
    1212      Raises:
    1213        httplib2.HttpLib2Error if a transport error has occured.
    1214        apiclient.errors.BatchError if the response is the wrong format.
    1215      """
    1216      message = MIMEMultipart('mixed')
    1217      # Message should not write out it's own headers.
    1218      setattr(message, '_write_headers', lambda self: None)
    1219
    1220      # Add all the individual requests.
    1221      for request_id in order:
    1222        request = requests[request_id]
    1223
    1224        msg = MIMENonMultipart('application', 'http')
    1225        msg['Content-Transfer-Encoding'] = 'binary'
    1226        msg['Content-ID'] = self._id_to_header(request_id)
    1227
    1228        body = self._serialize_request(request)
    1229        msg.set_payload(body)
    1230        message.attach(msg)
    1231
    1232      body = message.as_string()
    1233
    1234      headers = {}
    1235      headers['content-type'] = ('multipart/mixed; '
    1236                                 'boundary="%s"') % message.get_boundary()
    1237
    1238      resp, content = http.request(self._batch_uri, method='POST', body=body,
    1239                                   headers=headers)
    1240
    1241      if resp.status >= 300:
    1242        raise HttpError(resp, content, uri=self._batch_uri)
    1243
    1244      # Now break out the individual responses and store each one.
    1245      boundary, _ = content.split(None, 1)
    1246
    1247      # Prepend with a content-type header so FeedParser can handle it.
    1248      header = 'content-type: %s\r\n\r\n' % resp['content-type']
    1249      for_parser = header + content
    1250
    1251      parser = FeedParser()
    1252      parser.feed(for_parser)
    1253      mime_response = parser.close()
    1254
    1255      if not mime_response.is_multipart():
    1256        raise BatchError("Response not in multipart/mixed format.", resp=resp,
    1257                         content=content)
    1258
    1259      for part in mime_response.get_payload():
    1260        request_id = self._header_to_id(part['Content-ID'])
    1261        response, content = self._deserialize_response(part.get_payload())
    1262        self._responses[request_id] = (response, content)
    1263
    1264    @util.positional(1)
    1265 -  def execute(self, http=None):
    1266      """Execute all the requests as a single batched HTTP request.
    1267
    1268      Args:
    1269        http: httplib2.Http, an http object to be used in place of the one the
    1270          HttpRequest request object was constructed with. If one isn't supplied
    1271          then use a http object from the requests in this batch.
    1272
    1273      Returns:
    1274        None
    1275
    1276      Raises:
    1277        httplib2.HttpLib2Error if a transport error has occured.
    1278        apiclient.errors.BatchError if the response is the wrong format.
    1279      """
    1280
    1281      # If http is not supplied use the first valid one given in the requests.
    1282      if http is None:
    1283        for request_id in self._order:
    1284          request = self._requests[request_id]
    1285          if request is not None:
    1286            http = request.http
    1287            break
    1288
    1289      if http is None:
    1290        raise ValueError("Missing a valid http object.")
    1291
    1292      self._execute(http, self._order, self._requests)
    1293
    1294      # Loop over all the requests and check for 401s. For each 401 request the
    1295      # credentials should be refreshed and then sent again in a separate batch.
    1296      redo_requests = {}
    1297      redo_order = []
    1298
    1299      for request_id in self._order:
    1300        resp, content = self._responses[request_id]
    1301        if resp['status'] == '401':
    1302          redo_order.append(request_id)
    1303          request = self._requests[request_id]
    1304          self._refresh_and_apply_credentials(request, http)
    1305          redo_requests[request_id] = request
    1306
    1307      if redo_requests:
    1308        self._execute(http, redo_order, redo_requests)
    1309
    1310      # Now process all callbacks that are erroring, and raise an exception for
    1311      # ones that return a non-2xx response? Or add extra parameter to callback
    1312      # that contains an HttpError?
    1313
    1314      for request_id in self._order:
    1315        resp, content = self._responses[request_id]
    1316
    1317        request = self._requests[request_id]
    1318        callback = self._callbacks[request_id]
    1319
    1320        response = None
    1321        exception = None
    1322        try:
    1323          if resp.status >= 300:
    1324            raise HttpError(resp, content, uri=request.uri)
    1325          response = request.postproc(resp, content)
    1326        except HttpError, e:
    1327          exception = e
    1328
    1329        if callback is not None:
    1330          callback(request_id, response, exception)
    1331        if self._callback is not None:
    1332          self._callback(request_id, response, exception)
    1333
    1334
    1335 -class HttpRequestMock(object):
    1336    """Mock of HttpRequest.
    1337
    1338    Do not construct directly, instead use RequestMockBuilder.
    1339    """
    1340
    1341 -  def __init__(self, resp, content, postproc):
    1342      """Constructor for HttpRequestMock
    1343
    1344      Args:
    1345        resp: httplib2.Response, the response to emulate coming from the request
    1346        content: string, the response body
    1347        postproc: callable, the post processing function usually supplied by
    1348                  the model class. See model.JsonModel.response() as an example.
    1349      """
    1350      self.resp = resp
    1351      self.content = content
    1352      self.postproc = postproc
    1353      if resp is None:
    1354        self.resp = httplib2.Response({'status': 200, 'reason': 'OK'})
    1355      if 'reason' in self.resp:
    1356        self.resp.reason = self.resp['reason']
    1357
    1358 -  def execute(self, http=None):
    1359      """Execute the request.
    1360
    1361      Same behavior as HttpRequest.execute(), but the response is
    1362      mocked and not really from an HTTP request/response.
    1363      """
    1364      return self.postproc(self.resp, self.content)
    1365
    1366
    1367 -class RequestMockBuilder(object):
    1368    """A simple mock of HttpRequest
    1369
    1370      Pass in a dictionary to the constructor that maps request methodIds to
    1371      tuples of (httplib2.Response, content, opt_expected_body) that should be
    1372      returned when that method is called. None may also be passed in for the
    1373      httplib2.Response, in which case a 200 OK response will be generated.
    1374      If an opt_expected_body (str or dict) is provided, it will be compared to
    1375      the body and UnexpectedBodyError will be raised on inequality.
    1376
    1377      Example:
    1378        response = '{"data": {"id": "tag:google.c...'
    1379        requestBuilder = RequestMockBuilder(
    1380          {
    1381            'plus.activities.get': (None, response),
    1382          }
    1383        )
    1384        apiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder)
    1385
    1386      Methods that you do not supply a response for will return a
    1387      200 OK with an empty string as the response content or raise an excpetion
    1388      if check_unexpected is set to True. The methodId is taken from the rpcName
    1389      in the discovery document.
    1390
    1391      For more details see the project wiki.
    1392    """
    1393
    1394 -  def __init__(self, responses, check_unexpected=False):
    1395      """Constructor for RequestMockBuilder
    1396
    1397      The constructed object should be a callable object
    1398      that can replace the class HttpResponse.
    1399
    1400      responses - A dictionary that maps methodIds into tuples
    1401                  of (httplib2.Response, content). The methodId
    1402                  comes from the 'rpcName' field in the discovery
    1403                  document.
    1404      check_unexpected - A boolean setting whether or not UnexpectedMethodError
    1405                         should be raised on unsupplied method.
    1406      """
    1407      self.responses = responses
    1408      self.check_unexpected = check_unexpected
    1409
    1410 -  def __call__(self, http, postproc, uri, method='GET', body=None,
    1411                 headers=None, methodId=None, resumable=None):
    1412      """Implements the callable interface that discovery.build() expects
    1413      of requestBuilder, which is to build an object compatible with
    1414      HttpRequest.execute(). See that method for the description of the
    1415      parameters and the expected response.
    1416      """
    1417      if methodId in self.responses:
    1418        response = self.responses[methodId]
    1419        resp, content = response[:2]
    1420        if len(response) > 2:
    1421          # Test the body against the supplied expected_body.
    1422          expected_body = response[2]
    1423          if bool(expected_body) != bool(body):
    1424            # Not expecting a body and provided one
    1425            # or expecting a body and not provided one.
    1426            raise UnexpectedBodyError(expected_body, body)
    1427          if isinstance(expected_body, str):
    1428            expected_body = simplejson.loads(expected_body)
    1429          body = simplejson.loads(body)
    1430          if body != expected_body:
    1431            raise UnexpectedBodyError(expected_body, body)
    1432        return HttpRequestMock(resp, content, postproc)
    1433      elif self.check_unexpected:
    1434        raise UnexpectedMethodError(methodId=methodId)
    1435      else:
    1436        model = JsonModel(False)
    1437        return HttpRequestMock(None, '{}', model.response)
    1438
    1439
    1440 -class HttpMock(object):
    1441    """Mock of httplib2.Http"""
    1442
    1443 -  def __init__(self, filename=None, headers=None):
    1444      """
    1445      Args:
    1446        filename: string, absolute filename to read response from
    1447        headers: dict, header to return with response
    1448      """
    1449      if headers is None:
    1450        headers = {'status': '200 OK'}
    1451      if filename:
    1452        f = file(filename, 'r')
    1453        self.data = f.read()
    1454        f.close()
    1455      else:
    1456        self.data = None
    1457      self.response_headers = headers
    1458      self.headers = None
    1459      self.uri = None
    1460      self.method = None
    1461      self.body = None
    1462      self.headers = None
    1463
    1464
    1465 -  def request(self, uri,
    1466                method='GET',
    1467                body=None,
    1468                headers=None,
    1469                redirections=1,
    1470                connection_type=None):
    1471      self.uri = uri
    1472      self.method = method
    1473      self.body = body
    1474      self.headers = headers
    1475      return httplib2.Response(self.response_headers), self.data
    1476
    1477
    1478 -class HttpMockSequence(object):
    1479    """Mock of httplib2.Http
    1480
    1481    Mocks a sequence of calls to request returning different responses for each
    1482    call. Create an instance initialized with the desired response headers
    1483    and content and then use as if an httplib2.Http instance.
    1484
    1485      http = HttpMockSequence([
    1486        ({'status': '401'}, ''),
    1487        ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'),
    1488        ({'status': '200'}, 'echo_request_headers'),
    1489        ])
    1490      resp, content = http.request("http://examples.com")
    1491
    1492    There are special values you can pass in for content to trigger
    1493    behavours that are helpful in testing.
    1494
    1495    'echo_request_headers' means return the request headers in the response body
    1496    'echo_request_headers_as_json' means return the request headers in
    1497       the response body
    1498    'echo_request_body' means return the request body in the response body
    1499    'echo_request_uri' means return the request uri in the response body
    1500    """
    1501
    1502 -  def __init__(self, iterable):
    1503      """
    1504      Args:
    1505        iterable: iterable, a sequence of pairs of (headers, body)
    1506      """
    1507      self._iterable = iterable
    1508      self.follow_redirects = True
    1509
    1510 -  def request(self, uri,
    1511                method='GET',
    1512                body=None,
    1513                headers=None,
    1514                redirections=1,
    1515                connection_type=None):
    1516      resp, content = self._iterable.pop(0)
    1517      if content == 'echo_request_headers':
    1518        content = headers
    1519      elif content == 'echo_request_headers_as_json':
    1520        content = simplejson.dumps(headers)
    1521      elif content == 'echo_request_body':
    1522        if hasattr(body, 'read'):
    1523          content = body.read()
    1524        else:
    1525          content = body
    1526      elif content == 'echo_request_uri':
    1527        content = uri
    1528      return httplib2.Response(resp), content
    1529
    1530
    1531 -def set_user_agent(http, user_agent):
    1532    """Set the user-agent on every request.
    1533
    1534    Args:
    1535       http - An instance of httplib2.Http
    1536           or something that acts like it.
    1537       user_agent: string, the value for the user-agent header.
    1538
    1539    Returns:
    1540       A modified instance of http that was passed in.
    1541
    1542    Example:
    1543
    1544      h = httplib2.Http()
    1545      h = set_user_agent(h, "my-app-name/6.0")
    1546
    1547    Most of the time the user-agent will be set doing auth, this is for the rare
    1548    cases where you are accessing an unauthenticated endpoint.
    1549    """
    1550    request_orig = http.request
    1551
    1552    # The closure that will replace 'httplib2.Http.request'.
    1553    def new_request(uri, method='GET', body=None, headers=None,
    1554                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1555                    connection_type=None):
    1556      """Modify the request headers to add the user-agent."""
    1557      if headers is None:
    1558        headers = {}
    1559      if 'user-agent' in headers:
    1560        headers['user-agent'] = user_agent + ' ' + headers['user-agent']
    1561      else:
    1562        headers['user-agent'] = user_agent
    1563      resp, content = request_orig(uri, method, body, headers,
    1564                          redirections, connection_type)
    1565      return resp, content
    1566
    1567    http.request = new_request
    1568    return http
    1569
    1570
    1571 -def tunnel_patch(http):
    1572    """Tunnel PATCH requests over POST.
    1573    Args:
    1574       http - An instance of httplib2.Http
    1575           or something that acts like it.
    1576
    1577    Returns:
    1578       A modified instance of http that was passed in.
    1579
    1580    Example:
    1581
    1582      h = httplib2.Http()
    1583      h = tunnel_patch(h, "my-app-name/6.0")
    1584
    1585    Useful if you are running on a platform that doesn't support PATCH.
    1586    Apply this last if you are using OAuth 1.0, as changing the method
    1587    will result in a different signature.
    1588    """
    1589    request_orig = http.request
    1590
    1591    # The closure that will replace 'httplib2.Http.request'.
    1592    def new_request(uri, method='GET', body=None, headers=None,
    1593                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
    1594                    connection_type=None):
    1595      """Modify the request headers to add the user-agent."""
    1596      if headers is None:
    1597        headers = {}
    1598      if method == 'PATCH':
    1599        if 'oauth_token' in headers.get('authorization', ''):
    1600          logging.warning(
    1601              'OAuth 1.0 request made with Credentials after tunnel_patch.')
    1602        headers['x-http-method-override'] = "PATCH"
    1603        method = 'POST'
    1604      resp, content = request_orig(uri, method, body, headers,
    1605                          redirections, connection_type)
    1606      return resp, content
    1607
    1608    http.request = new_request
    1609    return http
    1610