From fd38036b9f1693f8f368851d40928bc5922ce606 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 28 Jun 2012 16:38:03 +0200 Subject: remove workarounds for missing next() and hashlib --- wsgitools/filters.py | 5 ----- 1 file changed, 5 deletions(-) (limited to 'wsgitools/filters.py') diff --git a/wsgitools/filters.py b/wsgitools/filters.py index 7ae1b69..4c7ff20 100644 --- a/wsgitools/filters.py +++ b/wsgitools/filters.py @@ -17,11 +17,6 @@ try: except ImportError: import StringIO as io -try: - next -except NameError: - def next(it): - return it.next() __all__.append("CloseableIterator") class CloseableIterator: -- cgit v1.2.3 From 53c7d892904c9f133d55f052a27dafd3911d0c5c Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 28 Jun 2012 17:23:06 +0200 Subject: provide py3 style __next__ methods --- wsgitools/filters.py | 4 +++- wsgitools/scgi/__init__.py | 4 +++- wsgitools/scgi/forkpool.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) (limited to 'wsgitools/filters.py') diff --git a/wsgitools/filters.py b/wsgitools/filters.py index 4c7ff20..4305c9d 100644 --- a/wsgitools/filters.py +++ b/wsgitools/filters.py @@ -35,7 +35,7 @@ class CloseableIterator: @rtype: gen() """ return self - def next(self): + def __next__(self): """iterator interface""" if not self.iterators: raise StopIteration @@ -44,6 +44,8 @@ class CloseableIterator: except StopIteration: self.iterators.pop(0) return next(self) + def next(self): + return self.__next__() __all__.append("CloseableList") class CloseableList(list): diff --git a/wsgitools/scgi/__init__.py b/wsgitools/scgi/__init__.py index cbe7a80..4e60b74 100644 --- a/wsgitools/scgi/__init__.py +++ b/wsgitools/scgi/__init__.py @@ -45,13 +45,15 @@ class FileWrapper: def __iter__(self): return self - def next(self): + def __next__(self): assert self.offset <= 0 self.offset = -1 data = self.filelike.read(self.blksize) if data: return data raise StopIteration + def next(self): + return self.__next__() def _convert_environ(environ, multithread=False, multiprocess=False, run_once=False): diff --git a/wsgitools/scgi/forkpool.py b/wsgitools/scgi/forkpool.py index 1bf0c6f..7cc6d18 100644 --- a/wsgitools/scgi/forkpool.py +++ b/wsgitools/scgi/forkpool.py @@ -135,7 +135,7 @@ class SocketFileWrapper: def __iter__(self): """see pep333""" return self - def next(self): + def __next__(self): """ see pep333 @raise socket.error: @@ -144,6 +144,8 @@ class SocketFileWrapper: if not data: raise StopIteration return data + def next(self): + return self.__next__() def flush(self): """see pep333""" pass -- cgit v1.2.3 From 7e2e9173b2afcc2a8dca9e6047d0b82ad70c9dff Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 28 Jun 2012 22:38:28 +0200 Subject: first part of bytes conversion Convert the request body data from str to bytes. This replaces all StringIOs with BytesIOs (removing backwards one more backwards compatibility). Also all character sequences involved in request bodies get a b"" prefix. The StaticContent application takes bytes instead of str (no difference for py2x). The GzipWSGIFilter needs a fixed as a truncate of a BytesIO does not rewind the stream position. --- test.py | 43 +++++++++++++++++++----------------------- wsgitools/applications.py | 6 +++--- wsgitools/authentication.py | 4 ++-- wsgitools/filters.py | 21 ++++++++------------- wsgitools/middlewares.py | 11 +++-------- wsgitools/scgi/asynchronous.py | 9 ++------- 6 files changed, 37 insertions(+), 57 deletions(-) (limited to 'wsgitools/filters.py') diff --git a/test.py b/test.py index f46a512..f2e8910 100755 --- a/test.py +++ b/test.py @@ -3,12 +3,7 @@ import unittest import doctest import wsgiref.validate -# Cannot use io module as it is broken in 2.6. -# Writing a str to a io.StringIO results in an exception. -try: - import cStringIO as io -except ImportError: - import StringIO as io +import io from hashlib import md5 import sys @@ -27,7 +22,7 @@ class Request: QUERY_STRING="") self.environ.update({ "wsgi.version": (1, 0), - "wsgi.input": io.StringIO(), + "wsgi.input": io.BytesIO(), "wsgi.errors": sys.stderr, "wsgi.url_scheme": "http", "wsgi.multithread": False, @@ -122,14 +117,14 @@ class Result: self.testcase.fail("header %s not found" % name) def get_data(self): - return "".join(self.writtendata) + "".join(self.returneddata) + return b"".join(self.writtendata) + b"".join(self.returneddata) from wsgitools import applications class StaticContentTest(unittest.TestCase): def setUp(self): self.app = applications.StaticContent( - "200 Found", [("Content-Type", "text/plain")], "nothing") + "200 Found", [("Content-Type", "text/plain")], b"nothing") self.req = Request(self) def testGet(self): @@ -146,7 +141,7 @@ class StaticContentTest(unittest.TestCase): class StaticFileTest(unittest.TestCase): def setUp(self): - self.app = applications.StaticFile(io.StringIO("success"), "200 Found", + self.app = applications.StaticFile(io.BytesIO(b"success"), "200 Found", [("Content-Type", "text/plain")]) self.req = Request(self) @@ -167,7 +162,7 @@ from wsgitools import digest class AuthDigestMiddlewareTest(unittest.TestCase): def setUp(self): self.staticapp = applications.StaticContent( - "200 Found", [("Content-Type", "text/plain")], "success") + "200 Found", [("Content-Type", "text/plain")], b"success") token_gen = digest.AuthTokenGenerator("foo", lambda _: "baz") self.app = digest.AuthDigestMiddleware( wsgiref.validate.validator(self.staticapp), token_gen) @@ -232,28 +227,28 @@ from wsgitools import middlewares def writing_application(environ, start_response): write = start_response("404 Not found", [("Content-Type", "text/plain")]) write = start_response("200 Ok", [("Content-Type", "text/plain")]) - write("first") - yield "" - yield "second" + write(b"first") + yield b"" + yield b"second" def write_only_application(environ, start_response): write = start_response("200 Ok", [("Content-Type", "text/plain")]) - write("first") - write("second") - yield "" + write(b"first") + write(b"second") + yield b"" class NoWriteCallableMiddlewareTest(unittest.TestCase): def testWrite(self): app = middlewares.NoWriteCallableMiddleware(writing_application) res = Request(self)(app) self.assertEqual(res.writtendata, []) - self.assertEqual("".join(res.returneddata), "firstsecond") + self.assertEqual(b"".join(res.returneddata), b"firstsecond") def testWriteOnly(self): app = middlewares.NoWriteCallableMiddleware(write_only_application) res = Request(self)(app) self.assertEqual(res.writtendata, []) - self.assertEqual("".join(res.returneddata), "firstsecond") + self.assertEqual(b"".join(res.returneddata), b"firstsecond") class StupidIO: """file-like without tell method, so StaticFile is not able to @@ -273,7 +268,7 @@ class StupidIO: class ContentLengthMiddlewareTest(unittest.TestCase): def setUp(self): - self.staticapp = applications.StaticFile(StupidIO("success"), + self.staticapp = applications.StaticFile(StupidIO(b"success"), "200 Found", [("Content-Type", "text/plain")]) self.app = middlewares.ContentLengthMiddleware(self.staticapp, maxstore=10) @@ -296,7 +291,7 @@ class ContentLengthMiddlewareTest(unittest.TestCase): class BasicAuthMiddlewareTest(unittest.TestCase): def setUp(self): self.staticapp = applications.StaticContent( - "200 Found", [("Content-Type", "text/plain")], "success") + "200 Found", [("Content-Type", "text/plain")], b"success") checkpw = middlewares.DictAuthChecker({"bar": "baz"}) self.app = middlewares.BasicAuthMiddleware( wsgiref.validate.validator(self.staticapp), checkpw) @@ -340,13 +335,13 @@ import gzip class GzipWSGIFilterTest(unittest.TestCase): def testSimple(self): app = applications.StaticContent("200 Found", - [("Content-Type", "text/plain")], "nothing") + [("Content-Type", "text/plain")], b"nothing") app = filters.WSGIFilterMiddleware(app, filters.GzipWSGIFilter) req = Request(self) req.environ["HTTP_ACCEPT_ENCODING"] = "gzip" res = req(app) - data = gzip.GzipFile(fileobj=io.StringIO(res.get_data())).read() - self.assertEqual(data, "nothing") + data = gzip.GzipFile(fileobj=io.BytesIO(res.get_data())).read() + self.assertEqual(data, b"nothing") def alltests(case): return unittest.TestLoader().loadTestsFromTestCase(case) diff --git a/wsgitools/applications.py b/wsgitools/applications.py index 8a02fe8..cdaf0ae 100644 --- a/wsgitools/applications.py +++ b/wsgitools/applications.py @@ -21,7 +21,7 @@ class StaticContent: @type headers: list @param headers: is a list of C{(header, value)} pairs being delivered as HTTP headers - @type content: basestring + @type content: bytes @param content: contains the data to be delivered to the client. It is either a string or some kind of iterable yielding strings. @type anymethod: boolean @@ -30,12 +30,12 @@ class StaticContent: """ assert isinstance(status, str) assert isinstance(headers, list) - assert isinstance(content, basestring) or hasattr(content, "__iter__") + assert isinstance(content, bytes) or hasattr(content, "__iter__") self.status = status self.headers = headers self.anymethod = anymethod length = -1 - if isinstance(content, basestring): + if isinstance(content, bytes): self.content = [content] length = len(content) else: diff --git a/wsgitools/authentication.py b/wsgitools/authentication.py index 963dc00..c076d7f 100644 --- a/wsgitools/authentication.py +++ b/wsgitools/authentication.py @@ -97,8 +97,8 @@ class AuthenticationMiddleware: @param exception: reason for the authentication failure """ status = "401 Authorization required" - html = "401 Authorization required" \ - "

401 Authorization required

" + html = b"401 Authorization required" \ + b"

401 Authorization required

" headers = [("Content-Type", "text/html"), self.www_authenticate(exception), ("Content-Length", str(len(html)))] diff --git a/wsgitools/filters.py b/wsgitools/filters.py index 4305c9d..6f90903 100644 --- a/wsgitools/filters.py +++ b/wsgitools/filters.py @@ -10,13 +10,7 @@ __all__ = [] import sys import time import gzip -# Cannot use io module as it is broken in 2.6. -# Writing a str to a io.StringIO results in an exception. -try: - import cStringIO as io -except ImportError: - import StringIO as io - +import io __all__.append("CloseableIterator") class CloseableIterator: @@ -397,7 +391,7 @@ class GzipWSGIFilter(BaseWSGIFilter): acceptenc = map(str.strip, acceptenc) if "gzip" in acceptenc: self.compress = True - self.sio = io.StringIO() + self.sio = io.BytesIO() self.gzip = gzip.GzipFile(fileobj=self.sio, mode="w") return environ def filter_header(self, headername, headervalue): @@ -431,6 +425,7 @@ class GzipWSGIFilter(BaseWSGIFilter): self.gzip.flush() data = self.sio.getvalue() self.sio.truncate(0) + self.sio.seek(0) return data def append_data(self): """BaseWSGIFilter interface @@ -446,7 +441,7 @@ class ReusableWSGIInputFilter(BaseWSGIFilter): """Make C{environ["wsgi.input"]} readable multiple times. Although this is not required by the standard it is sometimes desirable to read C{wsgi.input} multiple times. This filter will therefore replace that variable with a - C{StringIO} instance which provides a C{seek} method. + C{BytesIO} instance which provides a C{seek} method. """ @classmethod def creator(cls, maxrequestsize): @@ -457,14 +452,14 @@ class ReusableWSGIInputFilter(BaseWSGIFilter): adapter to eat this data.) @type maxrequestsize: int @param maxrequestsize: is the maximum number of bytes to store in the - C{StringIO} + C{BytesIO} """ return lambda:cls(maxrequestsize) def __init__(self, maxrequestsize=65536): """ReusableWSGIInputFilters constructor. @type maxrequestsize: int @param maxrequestsize: is the maximum number of bytes to store in the - C{StringIO}, see L{creator} + C{BytesIO}, see L{creator} """ BaseWSGIFilter.__init__(self) self.maxrequestsize = maxrequestsize @@ -474,12 +469,12 @@ class ReusableWSGIInputFilter(BaseWSGIFilter): @type environ: {str: str} """ - if isinstance(environ["wsgi.input"], io.StringIO): + if isinstance(environ["wsgi.input"], io.BytesIO): return environ # nothing to be done # XXX: is this really a good idea? use with care environ["wsgitools.oldinput"] = environ["wsgi.input"] - data = io.StringIO(environ["wsgi.input"].read(self.maxrequestsize)) + data = io.BytesIO(environ["wsgi.input"].read(self.maxrequestsize)) environ["wsgi.input"] = data return environ diff --git a/wsgitools/middlewares.py b/wsgitools/middlewares.py index dbf2020..e6ede9d 100644 --- a/wsgitools/middlewares.py +++ b/wsgitools/middlewares.py @@ -5,12 +5,7 @@ import sys import cgitb import binascii import collections -# Cannot use io module as it is broken in 2.6. -# Writing a str to a io.StringIO results in an exception. -try: - import cStringIO as io -except ImportError: - import StringIO as io +import io if sys.version_info[0] >= 3: def exc_info_for_raise(exc_info): @@ -60,7 +55,7 @@ __all__.append("NoWriteCallableMiddleware") class NoWriteCallableMiddleware: """This middleware wraps a wsgi application that needs the return value of C{start_response} function to a wsgi application that doesn't need one by - writing the data to a C{StringIO} and then making it be the first result + writing the data to a C{BytesIO} and then making it be the first result element.""" def __init__(self, app): """Wraps wsgi application app.""" @@ -72,7 +67,7 @@ class NoWriteCallableMiddleware: """ assert isinstance(environ, dict) todo = [None] - sio = io.StringIO() + sio = io.BytesIO() gotiterdata = False def write_calleable(data): assert not gotiterdata diff --git a/wsgitools/scgi/asynchronous.py b/wsgitools/scgi/asynchronous.py index 1dee283..7eb1a30 100644 --- a/wsgitools/scgi/asynchronous.py +++ b/wsgitools/scgi/asynchronous.py @@ -1,14 +1,9 @@ __all__ = [] import asyncore +import io import socket import sys -# Cannot use io module as it is broken in 2.6. -# Writing a str to a io.StringIO results in an exception. -try: - import cStringIO as io -except ImportError: - import StringIO as io import errno from wsgitools.scgi import _convert_environ, FileWrapper @@ -48,7 +43,7 @@ class SCGIConnection(asyncore.dispatcher): self.wsgiiterator = None # wsgi application iterator self.outheaders = () # headers to be sent # () -> unset, (..,..) -> set, True -> sent - self.body = io.StringIO() # request body + self.body = io.BytesIO() # request body def _try_send_headers(self): if self.outheaders != True: -- cgit v1.2.3 From 3f2f7f72a73caf087066c75d2e2b6e5ed908d34d Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Fri, 29 Jun 2012 09:26:09 +0200 Subject: fix more bytes related issues not covered by test.py * applications returned errors as str instead of bytes * filters documentation updated with bytes * various filters expecting str where bytes are passed * escape_string also needs to use bytes.isalnum instead of str.isalnum * middlewares injecting str where bytes are expected --- wsgitools/applications.py | 6 +++--- wsgitools/filters.py | 40 ++++++++++++++++------------------------ wsgitools/middlewares.py | 8 ++++---- 3 files changed, 23 insertions(+), 31 deletions(-) (limited to 'wsgitools/filters.py') diff --git a/wsgitools/applications.py b/wsgitools/applications.py index cdaf0ae..dfd8a2f 100644 --- a/wsgitools/applications.py +++ b/wsgitools/applications.py @@ -50,7 +50,7 @@ class StaticContent: assert isinstance(environ, dict) if environ["REQUEST_METHOD"].upper() not in ["GET", "HEAD"] and \ not self.anymethod: - resp = "Request method not implemented" + resp = b"Request method not implemented" start_response("501 Not Implemented", [("Content-length", str(len(resp)))]) return [resp] @@ -102,7 +102,7 @@ class StaticFile: assert isinstance(environ, dict) if environ["REQUEST_METHOD"].upper() not in ["GET", "HEAD"]: - resp = "Request method not implemented" + resp = b"Request method not implemented" start_response("501 Not Implemented", [("Content-length", str(len(resp)))]) return [resp] @@ -121,7 +121,7 @@ class StaticFile: size = stream.tell() stream.seek(0) except IOError: - resp = "File not found" + resp = b"File not found" start_response("404 File not found", [("Content-length", str(len(resp)))]) return [resp] diff --git a/wsgitools/filters.py b/wsgitools/filters.py index 6f90903..882a0bf 100644 --- a/wsgitools/filters.py +++ b/wsgitools/filters.py @@ -12,6 +12,8 @@ import time import gzip import io +from wsgitools.internal import str2bytes + __all__.append("CloseableIterator") class CloseableIterator: """Concatenating iterator with close attribute.""" @@ -122,15 +124,15 @@ class BaseWSGIFilter: """For each string that is either written by the C{write} callable or returned from the wrapped wsgi application this method is invoked. It must return a string. - @type data: str - @rtype: str + @type data: bytes + @rtype: bytes """ return data def append_data(self): """This function can be used to append data to the response. A list of strings or some kind of iterable yielding strings has to be returned. The default is to return an empty list. - @rtype: gen([str]) + @rtype: gen([bytes]) """ return [] def handle_close(self): @@ -152,7 +154,7 @@ class WSGIFilterMiddleware: def __call__(self, environ, start_response): """wsgi interface @type environ: {str, str} - @rtype: gen([str]) + @rtype: gen([bytes]) """ assert isinstance(environ, dict) reqfilter = self.filterclass() @@ -196,7 +198,7 @@ class WSGIFilterMiddleware: # default arguments. Also note that neither ' nor " are considered printable. # For escape_string to be reversible \ is also not considered printable. def escape_string(string, replacer=list(map( - lambda i: chr(i) if chr(i).isalnum() or + lambda i: chr(i) if str2bytes(chr(i)).isalnum() or chr(i) in '!#$%&()*+,-./:;<=>?@[]^_`{|}~ ' else r"\x%2.2x" % i, range(256)))): @@ -267,11 +269,7 @@ class RequestLogWSGIFilter(BaseWSGIFilter): self.status = status.split()[0] return status def filter_data(self, data): - """BaseWSGIFilter interface - @type data: str - @rtype: str - """ - assert isinstance(data, str) + assert isinstance(data, bytes) self.length += len(data) return data def handle_close(self): @@ -306,30 +304,31 @@ class TimerWSGIFilter(BaseWSGIFilter): def creator(cls, pattern): """Returns a function creating L{TimerWSGIFilter}s with a given pattern beeing a string of exactly eight bytes. - @type pattern: str + @type pattern: bytes """ return lambda:cls(pattern) - def __init__(self, pattern="?GenTime"): + def __init__(self, pattern=b"?GenTime"): """ @type pattern: str """ BaseWSGIFilter.__init__(self) + assert isinstance(pattern, bytes) self.pattern = pattern self.start = time.time() def filter_data(self, data): """BaseWSGIFilter interface - @type data: str - @rtype: str + @type data: bytes + @rtype: bytes """ if data == self.pattern: - return "%8.3g" % (time.time() - self.start) + return str2bytes("%8.3g" % (time.time() - self.start)) return data __all__.append("EncodeWSGIFilter") class EncodeWSGIFilter(BaseWSGIFilter): """Encodes all body data (no headers) with given charset. @note: This violates the wsgi standard as it requires unicode objects - whereas wsgi mandates the use of str. + whereas wsgi mandates the use of bytes. """ @classmethod def creator(cls, charset): @@ -347,7 +346,7 @@ class EncodeWSGIFilter(BaseWSGIFilter): def filter_data(self, data): """BaseWSGIFilter interface @type data: str - @rtype: str + @rtype: bytes """ return data.encode(self.charset) def filter_header(self, header, value): @@ -414,10 +413,6 @@ class GzipWSGIFilter(BaseWSGIFilter): headers.append(("Content-encoding", "gzip")) return headers def filter_data(self, data): - """BaseWSGIFilter interface - @type data: str - @rtype: str - """ if not self.compress: return data self.gzip.write(data) @@ -428,9 +423,6 @@ class GzipWSGIFilter(BaseWSGIFilter): self.sio.seek(0) return data def append_data(self): - """BaseWSGIFilter interface - @rtype: [str] - """ if not self.compress: return [] self.gzip.close() diff --git a/wsgitools/middlewares.py b/wsgitools/middlewares.py index 725deb1..4061d3b 100644 --- a/wsgitools/middlewares.py +++ b/wsgitools/middlewares.py @@ -33,7 +33,7 @@ class SubdirMiddleware: def __call__(self, environ, start_response): """wsgi interface @type environ: {str: str} - @rtype: gen([str]) + @rtype: gen([bytes]) """ assert isinstance(environ, dict) app = None @@ -65,7 +65,7 @@ class NoWriteCallableMiddleware: def __call__(self, environ, start_response): """wsgi interface @type environ: {str, str} - @rtype: gen([str]) + @rtype: gen([bytes]) """ assert isinstance(environ, dict) todo = [None] @@ -89,7 +89,7 @@ class NoWriteCallableMiddleware: ret = self.app(environ, modified_start_response) assert hasattr(ret, "__iter__") - first = "" + first = b"" if not isinstance(ret, list): ret = iter(ret) stopped = False @@ -171,7 +171,7 @@ class ContentLengthMiddleware: return ret ret = iter(ret) - first = "" + first = b"" stopped = False while not (first or stopped): try: -- cgit v1.2.3 From 27ed9839582c4fce9a0fff82281fb2e302be808e Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 6 Jun 2013 11:06:26 +0200 Subject: fix RequestLogWSGIFilterTest Clarify the type of the log file-like passed to RequestLogWSGIFilter. --- test.py | 5 ++++- wsgitools/filters.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'wsgitools/filters.py') diff --git a/test.py b/test.py index cf40b92..2d1d29d 100755 --- a/test.py +++ b/test.py @@ -377,7 +377,10 @@ class RequestLogWSGIFilterTest(unittest.TestCase): def testSimple(self): app = applications.StaticContent("200 Found", [("Content-Type", "text/plain")], b"nothing") - log = io.StringIO() + if isinstance("x", bytes): + log = io.BytesIO() + else: + log = io.StringIO() logfilter = filters.RequestLogWSGIFilter.creator(log) app = filters.WSGIFilterMiddleware(app, logfilter) req = Request(self) diff --git a/wsgitools/filters.py b/wsgitools/filters.py index 882a0bf..471c949 100644 --- a/wsgitools/filters.py +++ b/wsgitools/filters.py @@ -217,6 +217,9 @@ class RequestLogWSGIFilter(BaseWSGIFilter): """Returns a function creating L{RequestLogWSGIFilter}s on given log file. log has to be a file-like object. @type log: file-like + @param log: elements of type str are written to the log. That means in + Py3.X the contents are decoded and in Py2.X the log is assumed + to be encoded in latin1. This follows the spirit of WSGI. @type flush: bool @param flush: if True, invoke the flush method on log after each write invocation -- cgit v1.2.3