From 11e1fd2b0d9f2b46eb2e6a1f4cb985ec7d0d0e01 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Mon, 2 Jul 2012 08:17:20 +0200 Subject: added RequestLogWSGIFilterTest --- test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test.py b/test.py index 1acf5aa..5a96f93 100755 --- a/test.py +++ b/test.py @@ -2,6 +2,7 @@ import unittest import doctest +import re import wsgiref.validate # Cannot use io module as it is broken in 2.6. # Writing a str to a io.StringIO results in an exception. @@ -346,6 +347,22 @@ class BasicAuthMiddlewareTest(unittest.TestCase): from wsgitools import filters import gzip +class RequestLogWSGIFilterTest(unittest.TestCase): + def testSimple(self): + app = applications.StaticContent("200 Found", + [("Content-Type", "text/plain")], "nothing") + log = io.StringIO() + logfilter = filters.RequestLogWSGIFilter.creator(log) + app = filters.WSGIFilterMiddleware(app, logfilter) + req = Request(self) + req.environ["REMOTE_ADDR"] = "1.2.3.4" + req.environ["PATH_INFO"] = "/" + req.environ["HTTP_USER_AGENT"] = "wsgitools-test" + res = req(app) + logged = log.getvalue() + self.assert_(re.match(r'^1\.2\.3\.4 - - \[[^]]+\] "GET /" ' + r'200 7 - "wsgitools-test"', logged)) + class GzipWSGIFilterTest(unittest.TestCase): def testSimple(self): app = applications.StaticContent("200 Found", @@ -368,6 +385,7 @@ fullsuite.addTest(alltests(AuthDigestMiddlewareTest)) fullsuite.addTest(alltests(ContentLengthMiddlewareTest)) fullsuite.addTest(alltests(BasicAuthMiddlewareTest)) fullsuite.addTest(alltests(NoWriteCallableMiddlewareTest)) +fullsuite.addTest(alltests(RequestLogWSGIFilterTest)) fullsuite.addTest(alltests(GzipWSGIFilterTest)) if __name__ == "__main__": -- cgit v1.2.3 From 1bb2218f854e86229b76576850711c90a38a0bc9 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 1 Nov 2012 10:41:59 +0100 Subject: scgi.forkpool: reduce instance attributes The interface and port attributes are always used together. Combine them in order to reduce complexity. --- wsgitools/scgi/forkpool.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/wsgitools/scgi/forkpool.py b/wsgitools/scgi/forkpool.py index cdd50f0..64d93ef 100644 --- a/wsgitools/scgi/forkpool.py +++ b/wsgitools/scgi/forkpool.py @@ -206,8 +206,7 @@ class SCGIServer: """ assert hasattr(error, "write") self.wsgiapp = wsgiapp - self.port = port - self.interface = interface + self.bind_address = (interface, port) self.minworkers = minworkers self.maxworkers = maxworkers self.maxrequests = maxrequests @@ -237,7 +236,7 @@ class SCGIServer: if self.reusesocket is None: self.server = socket.socket() self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.server.bind((self.interface, self.port)) + self.server.bind(self.bind_address) self.server.listen(5) else: self.server = self.reusesocket -- cgit v1.2.3 From 36a72a53a62c3e86ed67852a6767d0ddb0a437b9 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 1 Nov 2012 11:04:53 +0100 Subject: scgi.forkpool: similarly drop the error attribute It can be stored inside the config attribute. --- wsgitools/scgi/forkpool.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wsgitools/scgi/forkpool.py b/wsgitools/scgi/forkpool.py index 64d93ef..5931deb 100644 --- a/wsgitools/scgi/forkpool.py +++ b/wsgitools/scgi/forkpool.py @@ -185,7 +185,7 @@ class SCGIServer: @param port: is the tcp port to listen on @type interface: str @param interface: is the interface to bind to (default: C{"localhost"}) - @param error: is a file-like object beeing passed as C{wsgi.error} in + @param error: is a file-like object beeing passed as C{wsgi.errors} in environ @type minworkers: int @param minworkers: is the number of worker processes to spawn @@ -210,8 +210,8 @@ class SCGIServer: self.minworkers = minworkers self.maxworkers = maxworkers self.maxrequests = maxrequests - self.config = config - self.error = error + self.config = config.copy() + self.config["wsgi.errors"] = error self.reusesocket = reusesocket self.server = None # becomes a socket # maps filedescriptors to WorkerStates @@ -438,7 +438,6 @@ class SCGIServer: _convert_environ(environ, multiprocess=True) sfw = SocketFileWrapper(con, int(environ["CONTENT_LENGTH"])) environ["wsgi.input"] = sfw - environ["wsgi.errors"] = self.error result = self.wsgiapp(environ, start_response) assert hasattr(result, "__iter__") -- cgit v1.2.3 From dd8148e11636d6c3792e8ceb03478048ae8eb571 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 1 Nov 2012 11:33:34 +0100 Subject: scgi.forkpool: fixed wrong assertion The forkpool server was incompatible with dumb generators. They only call start_response when being asked for the first output element, but the forkpool server was wrongly requiring start_response to be called before returning the iterator. --- wsgitools/scgi/forkpool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wsgitools/scgi/forkpool.py b/wsgitools/scgi/forkpool.py index 5931deb..52240f2 100644 --- a/wsgitools/scgi/forkpool.py +++ b/wsgitools/scgi/forkpool.py @@ -448,9 +448,9 @@ class SCGIServer: while sent > 0: sent = result.transfer(con) else: - assert response_head[0] is not None result_iter = iter(result) for data in result_iter: + assert response_head[0] is not None assert isinstance(data, str) dumbsend(data) if response_head[0] != True: -- cgit v1.2.3 From 576bd830cf77dae71b2cd10f0241667fe48930a6 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 1 Nov 2012 11:35:49 +0100 Subject: scgi.forkpool: implement RLIMIT_CPU The limit is only set on workers does not apply to the master. Upon reaching the soft limit the worker terminates after finished the current request. --- wsgitools/scgi/forkpool.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/wsgitools/scgi/forkpool.py b/wsgitools/scgi/forkpool.py index 52240f2..f27cd25 100644 --- a/wsgitools/scgi/forkpool.py +++ b/wsgitools/scgi/forkpool.py @@ -5,6 +5,10 @@ It works with multiple processes that are periodically cleaned up to prevent memory leaks having an impact to the system. """ +try: + import resource +except ImportError: + resource = None import socket import os import select @@ -178,7 +182,7 @@ class SCGIServer: def __init__(self, wsgiapp, port, interface="localhost", error=sys.stderr, minworkers=2, maxworkers=32, maxrequests=1000, config={}, - reusesocket=None): + reusesocket=None, cpulimit=None): """ @param wsgiapp: is the WSGI application to be run. @type port: int @@ -203,6 +207,12 @@ class SCGIServer: Instead use given socket as listen socket. The passed socket must be set up for accepting tcp connections (i.e. C{AF_INET}, C{SOCK_STREAM} with bind and listen called). + @type cpulimit: (int, int) + @param cpulimit: a pair of soft and hard cpu time limit in seconds. + This limit is installed for each worker using RLIMIT_CPU if + resource limits are available to this platform. After reaching + the soft limit workers will continue to process the current + request and then cleanly terminate. """ assert hasattr(error, "write") self.wsgiapp = wsgiapp @@ -213,6 +223,10 @@ class SCGIServer: self.config = config.copy() self.config["wsgi.errors"] = error self.reusesocket = reusesocket + # cpulimit changes meaning: + # master: None or a tuple denoting the limit to be configured. + # worker: boolean denoting whether the limit is reached. + self.cpulimit = cpulimit self.server = None # becomes a socket # maps filedescriptors to WorkerStates self.workers = {} @@ -302,6 +316,15 @@ class SCGIServer: else: self.running = False + def sigxcpuhandler(self, sig=None, stackframe=None): + """ + Signal hanlder function for the SIGXCUP signal. It is sent to a + worker when the soft RLIMIT_CPU is crossed. + @param sig: ignored for usage with signal.signal + @param stackframe: ignored for usage with signal.signal + """ + self.cpulimit = True + def spawnworker(self): """ internal! spawns a single worker @@ -317,6 +340,11 @@ class SCGIServer: worker.sock.close() del self.workers + if self.cpulimit and resource: + signal.signal(signal.SIGXCPU, self.sigxcpuhandler) + resource.setrlimit(resource.RLIMIT_CPU, self.cpulimit) + self.cpulimit = False + try: self.work(worksock) except socket.error: @@ -343,6 +371,8 @@ class SCGIServer: worksock.sendall('1') # tell server we're working self.process(con) worksock.sendall('0') # tell server we've finished + if self.cpulimit: + break def process(self, con): """ -- cgit v1.2.3 From d2f739d40ffe84538e0f743704d2f5c9331a1ba9 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Wed, 28 Nov 2012 16:08:20 +0100 Subject: added test for CachingMiddleware --- test.py | 54 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/test.py b/test.py index 5a96f93..3eb9553 100755 --- a/test.py +++ b/test.py @@ -48,21 +48,25 @@ class Request: """ @type key: str @type value: str + @returns: self """ self.environ[key] = value + return self def setmethod(self, request_method): """ @type request_method: str + @returns: self """ - self.setenv("REQUEST_METHOD", request_method) + return self.setenv("REQUEST_METHOD", request_method) def setheader(self, name, value): """ @type name: str @type value: str + @returns: self """ - self.setenv(name.upper().replace('-', '_'), value) + return self.setenv("HTTP_" + name.upper().replace('-', '_'), value) def copy(self): req = Request(self.testcase) @@ -190,14 +194,14 @@ class AuthDigestMiddlewareTest(unittest.TestCase): def test401garbage(self): req = self.req.copy() - req.setheader('http-authorization', 'Garbage') + req.setheader('Authorization', 'Garbage') res = req(self.app) res.status(401) res.header("WWW-Authenticate", lambda _: True) def test401digestgarbage(self): req = self.req.copy() - req.setheader('http-authorization', 'Digest ","') + req.setheader('Authorization', 'Digest ","') res = req(self.app) res.status(401) res.header("WWW-Authenticate", lambda _: True) @@ -211,7 +215,7 @@ class AuthDigestMiddlewareTest(unittest.TestCase): token = md5("bar:foo:%s" % password).hexdigest() other = md5("GET:").hexdigest() resp = md5("%s:%s:%s" % (token, nonce, other)).hexdigest() - req.setheader('http-authorization', 'Digest algorithm=md5,nonce="%s",' \ + req.setheader('Authorization', 'Digest algorithm=md5,nonce="%s",' \ 'uri=,username=bar,response="%s"' % (nonce, resp)) res = req(self.app) res.status(status) @@ -231,7 +235,7 @@ class AuthDigestMiddlewareTest(unittest.TestCase): token = md5("bar:foo:baz").hexdigest() other = md5("GET:").hexdigest() resp = md5("%s:%s:1:qux:auth:%s" % (token, nonce, other)).hexdigest() - req.setheader('http-authorization', 'Digest algorithm=md5,nonce="%s",' \ + req.setheader('Authorization', 'Digest algorithm=md5,nonce="%s",' \ 'uri=,username=bar,response="%s",qop=auth,nc=1,' \ 'cnonce=qux' % (nonce, resp)) res = req(self.app) @@ -303,6 +307,37 @@ class ContentLengthMiddlewareTest(unittest.TestCase): res.status("200 Found") res.header("Content-length", "7") +class CachingMiddlewareTest(unittest.TestCase): + def setUp(self): + self.cached = middlewares.CachingMiddleware(self.app) + self.accessed = dict() + + def app(self, environ, start_response): + count = self.accessed.get(environ["SCRIPT_NAME"], 0) + 1 + self.accessed[environ["SCRIPT_NAME"]] = count + headers = [("Content-Type", "text/plain")] + if "maxage0" in environ["SCRIPT_NAME"]: + headers.append(("Cache-Control", "max-age=0")) + start_response("200 Found", headers) + return ["%d" % count] + + def testCache(self): + res = Request(self)(self.cached) + res.status(200) + self.assertEqual(res.get_data(), "1") + res = Request(self)(self.cached) + res.status(200) + self.assertEqual(res.get_data(), "1") + + def testNoCache(self): + res = Request(self)(self.cached) + res.status(200) + self.assertEqual(res.get_data(), "1") + res = Request(self).setheader( + "Cache-Control", "max-age=0")(self.cached) + res.status(200) + self.assertEqual(res.get_data(), "2") + class BasicAuthMiddlewareTest(unittest.TestCase): def setUp(self): self.staticapp = applications.StaticContent( @@ -319,14 +354,14 @@ class BasicAuthMiddlewareTest(unittest.TestCase): def test401garbage(self): req = self.req.copy() - req.setheader('http-authorization', 'Garbage') + req.setheader('Authorization', 'Garbage') res = req(self.app) res.status(401) res.header("WWW-Authenticate", lambda _: True) def test401basicgarbage(self): req = self.req.copy() - req.setheader('http-authorization', 'Basic ()') + req.setheader('Authorization', 'Basic ()') res = req(self.app) res.status(401) res.header("WWW-Authenticate", lambda _: True) @@ -334,7 +369,7 @@ class BasicAuthMiddlewareTest(unittest.TestCase): def doauth(self, password="baz", status=200): req = self.req.copy() token = ("bar:%s" % password).encode("base64").strip() - req.setheader('http-authorization', 'Basic %s' % token) + req.setheader('Authorization', 'Basic %s' % token) res = req(self.app) res.status(status) @@ -383,6 +418,7 @@ fullsuite.addTest(alltests(StaticContentTest)) fullsuite.addTest(alltests(StaticFileTest)) fullsuite.addTest(alltests(AuthDigestMiddlewareTest)) fullsuite.addTest(alltests(ContentLengthMiddlewareTest)) +fullsuite.addTest(alltests(CachingMiddlewareTest)) fullsuite.addTest(alltests(BasicAuthMiddlewareTest)) fullsuite.addTest(alltests(NoWriteCallableMiddlewareTest)) fullsuite.addTest(alltests(RequestLogWSGIFilterTest)) -- cgit v1.2.3 From 30e8af066f3a091cf58443b8e45068c55bf0d68b Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Sun, 10 Mar 2013 14:20:25 +0100 Subject: forkpool: add a per-request timelimit --- wsgitools/scgi/forkpool.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/wsgitools/scgi/forkpool.py b/wsgitools/scgi/forkpool.py index f27cd25..514094f 100644 --- a/wsgitools/scgi/forkpool.py +++ b/wsgitools/scgi/forkpool.py @@ -182,7 +182,7 @@ class SCGIServer: def __init__(self, wsgiapp, port, interface="localhost", error=sys.stderr, minworkers=2, maxworkers=32, maxrequests=1000, config={}, - reusesocket=None, cpulimit=None): + reusesocket=None, cpulimit=None, timelimit=None): """ @param wsgiapp: is the WSGI application to be run. @type port: int @@ -213,6 +213,10 @@ class SCGIServer: resource limits are available to this platform. After reaching the soft limit workers will continue to process the current request and then cleanly terminate. + @type timelimit: int + @param timelimit: The maximum number of wall clock seconds processing + a request should take. If this is specified, an alarm timer is + installed and the default action is to kill the worker. """ assert hasattr(error, "write") self.wsgiapp = wsgiapp @@ -227,6 +231,7 @@ class SCGIServer: # master: None or a tuple denoting the limit to be configured. # worker: boolean denoting whether the limit is reached. self.cpulimit = cpulimit + self.timelimit = timelimit self.server = None # becomes a socket # maps filedescriptors to WorkerStates self.workers = {} @@ -369,7 +374,11 @@ class SCGIServer: (con, addr) = self.server.accept() # we cannot handle socket.errors here. worksock.sendall('1') # tell server we're working + if self.timelimit: + signal.alarm(self.timelimit) self.process(con) + if self.timelimit: + signal.alarm(0) worksock.sendall('0') # tell server we've finished if self.cpulimit: break -- cgit v1.2.3 From 9e30a7afad24061d4dd2da4546f401cc2371c5e0 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Mon, 11 Mar 2013 16:23:07 +0100 Subject: MANIFEST.in: include the test suite --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index d09c3ce..88ba387 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include LICENSE include epydoc.conf +include test.py -- cgit v1.2.3 From a91f557fdcbe3692a46351b97834928b17613ccf Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Tue, 12 Mar 2013 21:37:31 +0100 Subject: update Python versions in README --- README | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README b/README index fef9f25..1322692 100644 --- a/README +++ b/README @@ -2,13 +2,12 @@ The software should be usable by reading the docstrings. If you think that certain features are missing or you found a bug, don't hesitate to ask me via mail! -Supported Python versions currently are 2.5 and 2.6. It might still support +Supported Python versions currently are 2.5, 2.6 and 2.7. It might still support 2.4, but I did not test that. 3.x is currently not supported, because the -syntax is incompatible. However the fixes needed to make the code usable -with 3.x should be small. Eventually there will be a 3.x branch. +syntax is incompatible. There is a branch for for 3.x support. Installation should be easy using setup.py. I recommend running the test suite by invoking "python test.py" from the source tree to spot problems early. This -is especially true if you try Python versions other than 2.5 and 2.6. +is especially true if you try Python versions other than 2.6. Helmut Grohne -- cgit v1.2.3 From 928ded8d010b8591095f97b3e094cbefbad33e68 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Tue, 12 Mar 2013 21:38:07 +0100 Subject: extend copyright --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index caef583..daf0179 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ Applies to all files in this repository: -Copyright (C) 2007-2008 Helmut Grohne +Copyright (C) 2007-2013 Helmut Grohne This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by -- cgit v1.2.3 From ed6d6c8f06404489ba2301955c8e6f82f8f4f454 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 14 Mar 2013 13:44:24 +0100 Subject: release version 0.2.4 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 777ef6b..13f0992 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from distutils.core import setup setup(name="wsgitools", - version="0.2.3", + version="0.2.4", description="a set of tools working with WSGI (see PEP 333)", author="Helmut Grohne", author_email="helmut@subdivi.de", @@ -12,7 +12,7 @@ setup(name="wsgitools", license="GPL", keywords=["wsgi", "pep333", "scgi"], classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Environment :: No Input/Output (Daemon)", "Environment :: Web Environment", "Intended Audience :: Developers", -- cgit v1.2.3