Merge httpdownloader-fixes-2971

Author: paulegan, exarkun Reviewer: glyph Fixes: twisted#2971 Implement cookie, redirect limits, and timeouts for `downloadPage`. These are the same as the features already available for `getPage`. git-svn-id: svn://svn.twistedmatrix.com/svn/Twisted/trunk@26040 bbbe8e31-12d6-0310-92fd-ac37d47ddeeb
ardy30 · Jan 16, 2009 · f7820a6 · f7820a6
1 parent 878cb1e
commit f7820a6
Show file tree

Hide file tree

Showing 3 changed files with 112 additions and 12 deletions.
diff --git a/LICENSE b/LICENSE
@@ -1,4 +1,4 @@
-Copyright (c) 2001-2008
+Copyright (c) 2001-2009
 Allen Short
 Andrew Bennetts
 Apple Computer, Inc.

diff --git a/twisted/web/client.py b/twisted/web/client.py
@@ -1,5 +1,5 @@
 # -*- test-case-name: twisted.web.test.test_webclient -*-
-# Copyright (c) 2001-2008 Twisted Matrix Laboratories.
+# Copyright (c) 2001-2009 Twisted Matrix Laboratories.
 # See LICENSE for details.
 
 """
@@ -323,7 +323,9 @@ class HTTPDownloader(HTTPClientFactory):
 
     def __init__(self, url, fileOrName,
                  method='GET', postdata=None, headers=None,
-                 agent="Twisted client", supportPartial=0):
+                 agent="Twisted client", supportPartial=0,
+                 timeout=0, cookies=None, followRedirect=1,
+                 redirectLimit=20):
         self.requestedPartial = 0
         if isinstance(fileOrName, types.StringTypes):
             self.fileName = fileOrName
@@ -337,11 +339,14 @@ def __init__(self, url, fileOrName,
                     headers["range"] = "bytes=%d-" % fileLength
         else:
             self.file = fileOrName
-        HTTPClientFactory.__init__(self, url, method=method, postdata=postdata, headers=headers, agent=agent)
-        self.deferred = defer.Deferred()
-        self.waiting = 1
+        HTTPClientFactory.__init__(
+            self, url, method=method, postdata=postdata, headers=headers,
+            agent=agent, timeout=timeout, cookies=cookies,
+            followRedirect=followRedirect, redirectLimit=redirectLimit)
+
 
     def gotHeaders(self, headers):
+        HTTPClientFactory.gotHeaders(self, headers)
         if self.requestedPartial:
             contentRange = headers.get("content-range", None)
             if not contentRange:
@@ -369,7 +374,6 @@ def pageStart(self, partialContent):
         if partialContent and not self.requestedPartial:
             raise ValueError, "we shouldn't get partial content response if we didn't want it!"
         if self.waiting:
-            self.waiting = 0
             try:
                 if not self.file:
                     self.file = self.openFile(partialContent)
@@ -388,6 +392,7 @@ def pagePart(self, data):
             self.deferred.errback(failure.Failure())
 
     def pageEnd(self):
+        self.waiting = 0
         if not self.file:
             return
         try:

diff --git a/twisted/web/test/test_webclient.py b/twisted/web/test/test_webclient.py
@@ -1,11 +1,11 @@
-# Copyright (c) 2001-2008 Twisted Matrix Laboratories.
+# Copyright (c) 2001-2009 Twisted Matrix Laboratories.
 # See LICENSE for details.
 
 """
 Tests for L{twisted.web.client}.
 """
 
-import os
+import StringIO, os
 
 from urlparse import urlparse
 
@@ -23,7 +23,17 @@
 
 
 class ForeverTakingResource(resource.Resource):
+    """
+    L{ForeverTakingResource} is a resource which never finishes responding
+    to requests.
+    """
+    def __init__(self, write=False):
+        resource.Resource.__init__(self)
+        self._write = write
+
     def render(self, request):
+        if self._write:
+            request.write('some bytes')
         return server.NOT_DONE_YET
 
 
@@ -211,11 +221,13 @@ def setUp(self):
         self.infiniteRedirectResource = CountingRedirect("/infiniteRedirect")
         r.putChild("infiniteRedirect", self.infiniteRedirectResource)
         r.putChild("wait", ForeverTakingResource())
+        r.putChild("write-then-wait", ForeverTakingResource(write=True))
         r.putChild("error", ErrorResource())
         r.putChild("nolength", NoLengthResource())
         r.putChild("host", HostHeaderResource())
         r.putChild("payload", PayloadResource())
         r.putChild("broken", BrokenDownloadResource())
+        r.putChild("cookiemirror", CookieMirrorResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.port = self._listen(self.wrapper)
@@ -409,13 +421,13 @@ def test_infiniteRedirection(self):
         page request fails with L{InfiniteRedirection}.
         """
         def checkRedirectCount(*a):
-            self.assertEquals(f._redirectCount, 20)
-            self.assertEquals(self.infiniteRedirectResource.count, 20)
+            self.assertEquals(f._redirectCount, 13)
+            self.assertEquals(self.infiniteRedirectResource.count, 13)
 
         f = client._makeGetterFactory(
             self.getURL('infiniteRedirect'),
             client.HTTPClientFactory,
-            redirectLimit=20)
+            redirectLimit=13)
         d = self.assertFailure(f.deferred, error.InfiniteRedirection)
         d.addCallback(checkRedirectCount)
         return d
@@ -461,6 +473,89 @@ def _cbPartialTest(self, ignored, expectedData, filename):
         bytes = file(filename, "rb").read()
         self.assertEquals(bytes, expectedData)
 
+
+    def test_downloadTimeout(self):
+        """
+        If the timeout indicated by the C{timeout} parameter to
+        L{client.HTTPDownloader.__init__} elapses without the complete response
+        being received, the L{defer.Deferred} returned by
+        L{client.downloadPage} fires with a L{Failure} wrapping a
+        L{defer.TimeoutError}.
+        """
+        # Verify the behavior if no bytes are ever written.
+        first = client.downloadPage(
+            self.getURL("wait"),
+            self.mktemp(), timeout=0.01)
+
+        # Verify the behavior if some bytes are written but then the request
+        # never completes.
+        second = client.downloadPage(
+            self.getURL("write-then-wait"),
+            self.mktemp(), timeout=0.01)
+
+        return defer.gatherResults([
+            self.assertFailure(first, defer.TimeoutError),
+            self.assertFailure(second, defer.TimeoutError)])
+
+
+    def test_downloadHeaders(self):
+        """
+        After L{client.HTTPDownloader.deferred} fires, the
+        L{client.HTTPDownloader} instance's C{status} and C{response_headers}
+        attributes are populated with the values from the response.
+        """
+        def checkHeaders(factory):
+            self.assertEquals(factory.status, '200')
+            self.assertEquals(factory.response_headers['content-type'][0], 'text/html')
+            self.assertEquals(factory.response_headers['content-length'][0], '10')
+            os.unlink(factory.fileName)
+        factory = client._makeGetterFactory(
+            self.getURL('file'),
+            client.HTTPDownloader,
+            fileOrName=self.mktemp())
+        return factory.deferred.addCallback(lambda _: checkHeaders(factory))
+
+
+    def test_downloadCookies(self):
+        """
+        The C{cookies} dict passed to the L{client.HTTPDownloader}
+        initializer is used to populate the I{Cookie} header included in the
+        request sent to the server.
+        """
+        output = self.mktemp()
+        factory = client._makeGetterFactory(
+            self.getURL('cookiemirror'),
+            client.HTTPDownloader,
+            fileOrName=output,
+            cookies={'foo': 'bar'})
+        def cbFinished(ignored):
+            self.assertEqual(
+                FilePath(output).getContent(),
+                "[('foo', 'bar')]")
+        factory.deferred.addCallback(cbFinished)
+        return factory.deferred
+
+
+    def test_downloadRedirectLimit(self):
+        """
+        When more than C{redirectLimit} HTTP redirects are encountered, the
+        page request fails with L{InfiniteRedirection}.
+        """
+        def checkRedirectCount(*a):
+            self.assertEquals(f._redirectCount, 7)
+            self.assertEquals(self.infiniteRedirectResource.count, 7)
+
+        f = client._makeGetterFactory(
+            self.getURL('infiniteRedirect'),
+            client.HTTPDownloader,
+            fileOrName=self.mktemp(),
+            redirectLimit=7)
+        d = self.assertFailure(f.deferred, error.InfiniteRedirection)
+        d.addCallback(checkRedirectCount)
+        return d
+
+
+
 class WebClientSSLTestCase(WebClientTestCase):
     def _listen(self, site):
         from twisted import test