From 7e2e9173b2afcc2a8dca9e6047d0b82ad70c9dff Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 28 Jun 2012 22:38:28 +0200 Subject: first part of bytes conversion Convert the request body data from str to bytes. This replaces all StringIOs with BytesIOs (removing backwards one more backwards compatibility). Also all character sequences involved in request bodies get a b"" prefix. The StaticContent application takes bytes instead of str (no difference for py2x). The GzipWSGIFilter needs a fixed as a truncate of a BytesIO does not rewind the stream position. --- test.py | 43 +++++++++++++++++++----------------------- wsgitools/applications.py | 6 +++--- wsgitools/authentication.py | 4 ++-- wsgitools/filters.py | 21 ++++++++------------- wsgitools/middlewares.py | 11 +++-------- wsgitools/scgi/asynchronous.py | 9 ++------- 6 files changed, 37 insertions(+), 57 deletions(-) diff --git a/test.py b/test.py index f46a512..f2e8910 100755 --- a/test.py +++ b/test.py @@ -3,12 +3,7 @@ import unittest import doctest import wsgiref.validate -# Cannot use io module as it is broken in 2.6. -# Writing a str to a io.StringIO results in an exception. -try: - import cStringIO as io -except ImportError: - import StringIO as io +import io from hashlib import md5 import sys @@ -27,7 +22,7 @@ class Request: QUERY_STRING="") self.environ.update({ "wsgi.version": (1, 0), - "wsgi.input": io.StringIO(), + "wsgi.input": io.BytesIO(), "wsgi.errors": sys.stderr, "wsgi.url_scheme": "http", "wsgi.multithread": False, @@ -122,14 +117,14 @@ class Result: self.testcase.fail("header %s not found" % name) def get_data(self): - return "".join(self.writtendata) + "".join(self.returneddata) + return b"".join(self.writtendata) + b"".join(self.returneddata) from wsgitools import applications class StaticContentTest(unittest.TestCase): def setUp(self): self.app = applications.StaticContent( - "200 Found", [("Content-Type", "text/plain")], "nothing") + "200 Found", [("Content-Type", "text/plain")], b"nothing") self.req = Request(self) def testGet(self): @@ -146,7 +141,7 @@ class StaticContentTest(unittest.TestCase): class StaticFileTest(unittest.TestCase): def setUp(self): - self.app = applications.StaticFile(io.StringIO("success"), "200 Found", + self.app = applications.StaticFile(io.BytesIO(b"success"), "200 Found", [("Content-Type", "text/plain")]) self.req = Request(self) @@ -167,7 +162,7 @@ from wsgitools import digest class AuthDigestMiddlewareTest(unittest.TestCase): def setUp(self): self.staticapp = applications.StaticContent( - "200 Found", [("Content-Type", "text/plain")], "success") + "200 Found", [("Content-Type", "text/plain")], b"success") token_gen = digest.AuthTokenGenerator("foo", lambda _: "baz") self.app = digest.AuthDigestMiddleware( wsgiref.validate.validator(self.staticapp), token_gen) @@ -232,28 +227,28 @@ from wsgitools import middlewares def writing_application(environ, start_response): write = start_response("404 Not found", [("Content-Type", "text/plain")]) write = start_response("200 Ok", [("Content-Type", "text/plain")]) - write("first") - yield "" - yield "second" + write(b"first") + yield b"" + yield b"second" def write_only_application(environ, start_response): write = start_response("200 Ok", [("Content-Type", "text/plain")]) - write("first") - write("second") - yield "" + write(b"first") + write(b"second") + yield b"" class NoWriteCallableMiddlewareTest(unittest.TestCase): def testWrite(self): app = middlewares.NoWriteCallableMiddleware(writing_application) res = Request(self)(app) self.assertEqual(res.writtendata, []) - self.assertEqual("".join(res.returneddata), "firstsecond") + self.assertEqual(b"".join(res.returneddata), b"firstsecond") def testWriteOnly(self): app = middlewares.NoWriteCallableMiddleware(write_only_application) res = Request(self)(app) self.assertEqual(res.writtendata, []) - self.assertEqual("".join(res.returneddata), "firstsecond") + self.assertEqual(b"".join(res.returneddata), b"firstsecond") class StupidIO: """file-like without tell method, so StaticFile is not able to @@ -273,7 +268,7 @@ class StupidIO: class ContentLengthMiddlewareTest(unittest.TestCase): def setUp(self): - self.staticapp = applications.StaticFile(StupidIO("success"), + self.staticapp = applications.StaticFile(StupidIO(b"success"), "200 Found", [("Content-Type", "text/plain")]) self.app = middlewares.ContentLengthMiddleware(self.staticapp, maxstore=10) @@ -296,7 +291,7 @@ class ContentLengthMiddlewareTest(unittest.TestCase): class BasicAuthMiddlewareTest(unittest.TestCase): def setUp(self): self.staticapp = applications.StaticContent( - "200 Found", [("Content-Type", "text/plain")], "success") + "200 Found", [("Content-Type", "text/plain")], b"success") checkpw = middlewares.DictAuthChecker({"bar": "baz"}) self.app = middlewares.BasicAuthMiddleware( wsgiref.validate.validator(self.staticapp), checkpw) @@ -340,13 +335,13 @@ import gzip class GzipWSGIFilterTest(unittest.TestCase): def testSimple(self): app = applications.StaticContent("200 Found", - [("Content-Type", "text/plain")], "nothing") + [("Content-Type", "text/plain")], b"nothing") app = filters.WSGIFilterMiddleware(app, filters.GzipWSGIFilter) req = Request(self) req.environ["HTTP_ACCEPT_ENCODING"] = "gzip" res = req(app) - data = gzip.GzipFile(fileobj=io.StringIO(res.get_data())).read() - self.assertEqual(data, "nothing") + data = gzip.GzipFile(fileobj=io.BytesIO(res.get_data())).read() + self.assertEqual(data, b"nothing") def alltests(case): return unittest.TestLoader().loadTestsFromTestCase(case) diff --git a/wsgitools/applications.py b/wsgitools/applications.py index 8a02fe8..cdaf0ae 100644 --- a/wsgitools/applications.py +++ b/wsgitools/applications.py @@ -21,7 +21,7 @@ class StaticContent: @type headers: list @param headers: is a list of C{(header, value)} pairs being delivered as HTTP headers - @type content: basestring + @type content: bytes @param content: contains the data to be delivered to the client. It is either a string or some kind of iterable yielding strings. @type anymethod: boolean @@ -30,12 +30,12 @@ class StaticContent: """ assert isinstance(status, str) assert isinstance(headers, list) - assert isinstance(content, basestring) or hasattr(content, "__iter__") + assert isinstance(content, bytes) or hasattr(content, "__iter__") self.status = status self.headers = headers self.anymethod = anymethod length = -1 - if isinstance(content, basestring): + if isinstance(content, bytes): self.content = [content] length = len(content) else: diff --git a/wsgitools/authentication.py b/wsgitools/authentication.py index 963dc00..c076d7f 100644 --- a/wsgitools/authentication.py +++ b/wsgitools/authentication.py @@ -97,8 +97,8 @@ class AuthenticationMiddleware: @param exception: reason for the authentication failure """ status = "401 Authorization required" - html = "401 Authorization required" \ - "

401 Authorization required

" + html = b"401 Authorization required" \ + b"

401 Authorization required

" headers = [("Content-Type", "text/html"), self.www_authenticate(exception), ("Content-Length", str(len(html)))] diff --git a/wsgitools/filters.py b/wsgitools/filters.py index 4305c9d..6f90903 100644 --- a/wsgitools/filters.py +++ b/wsgitools/filters.py @@ -10,13 +10,7 @@ __all__ = [] import sys import time import gzip -# Cannot use io module as it is broken in 2.6. -# Writing a str to a io.StringIO results in an exception. -try: - import cStringIO as io -except ImportError: - import StringIO as io - +import io __all__.append("CloseableIterator") class CloseableIterator: @@ -397,7 +391,7 @@ class GzipWSGIFilter(BaseWSGIFilter): acceptenc = map(str.strip, acceptenc) if "gzip" in acceptenc: self.compress = True - self.sio = io.StringIO() + self.sio = io.BytesIO() self.gzip = gzip.GzipFile(fileobj=self.sio, mode="w") return environ def filter_header(self, headername, headervalue): @@ -431,6 +425,7 @@ class GzipWSGIFilter(BaseWSGIFilter): self.gzip.flush() data = self.sio.getvalue() self.sio.truncate(0) + self.sio.seek(0) return data def append_data(self): """BaseWSGIFilter interface @@ -446,7 +441,7 @@ class ReusableWSGIInputFilter(BaseWSGIFilter): """Make C{environ["wsgi.input"]} readable multiple times. Although this is not required by the standard it is sometimes desirable to read C{wsgi.input} multiple times. This filter will therefore replace that variable with a - C{StringIO} instance which provides a C{seek} method. + C{BytesIO} instance which provides a C{seek} method. """ @classmethod def creator(cls, maxrequestsize): @@ -457,14 +452,14 @@ class ReusableWSGIInputFilter(BaseWSGIFilter): adapter to eat this data.) @type maxrequestsize: int @param maxrequestsize: is the maximum number of bytes to store in the - C{StringIO} + C{BytesIO} """ return lambda:cls(maxrequestsize) def __init__(self, maxrequestsize=65536): """ReusableWSGIInputFilters constructor. @type maxrequestsize: int @param maxrequestsize: is the maximum number of bytes to store in the - C{StringIO}, see L{creator} + C{BytesIO}, see L{creator} """ BaseWSGIFilter.__init__(self) self.maxrequestsize = maxrequestsize @@ -474,12 +469,12 @@ class ReusableWSGIInputFilter(BaseWSGIFilter): @type environ: {str: str} """ - if isinstance(environ["wsgi.input"], io.StringIO): + if isinstance(environ["wsgi.input"], io.BytesIO): return environ # nothing to be done # XXX: is this really a good idea? use with care environ["wsgitools.oldinput"] = environ["wsgi.input"] - data = io.StringIO(environ["wsgi.input"].read(self.maxrequestsize)) + data = io.BytesIO(environ["wsgi.input"].read(self.maxrequestsize)) environ["wsgi.input"] = data return environ diff --git a/wsgitools/middlewares.py b/wsgitools/middlewares.py index dbf2020..e6ede9d 100644 --- a/wsgitools/middlewares.py +++ b/wsgitools/middlewares.py @@ -5,12 +5,7 @@ import sys import cgitb import binascii import collections -# Cannot use io module as it is broken in 2.6. -# Writing a str to a io.StringIO results in an exception. -try: - import cStringIO as io -except ImportError: - import StringIO as io +import io if sys.version_info[0] >= 3: def exc_info_for_raise(exc_info): @@ -60,7 +55,7 @@ __all__.append("NoWriteCallableMiddleware") class NoWriteCallableMiddleware: """This middleware wraps a wsgi application that needs the return value of C{start_response} function to a wsgi application that doesn't need one by - writing the data to a C{StringIO} and then making it be the first result + writing the data to a C{BytesIO} and then making it be the first result element.""" def __init__(self, app): """Wraps wsgi application app.""" @@ -72,7 +67,7 @@ class NoWriteCallableMiddleware: """ assert isinstance(environ, dict) todo = [None] - sio = io.StringIO() + sio = io.BytesIO() gotiterdata = False def write_calleable(data): assert not gotiterdata diff --git a/wsgitools/scgi/asynchronous.py b/wsgitools/scgi/asynchronous.py index 1dee283..7eb1a30 100644 --- a/wsgitools/scgi/asynchronous.py +++ b/wsgitools/scgi/asynchronous.py @@ -1,14 +1,9 @@ __all__ = [] import asyncore +import io import socket import sys -# Cannot use io module as it is broken in 2.6. -# Writing a str to a io.StringIO results in an exception. -try: - import cStringIO as io -except ImportError: - import StringIO as io import errno from wsgitools.scgi import _convert_environ, FileWrapper @@ -48,7 +43,7 @@ class SCGIConnection(asyncore.dispatcher): self.wsgiiterator = None # wsgi application iterator self.outheaders = () # headers to be sent # () -> unset, (..,..) -> set, True -> sent - self.body = io.StringIO() # request body + self.body = io.BytesIO() # request body def _try_send_headers(self): if self.outheaders != True: -- cgit v1.2.3