Create stand alone programs with full ZCA
Project description
Creating runners
The gocept.runner package allows it to easily create small, long running scripts which interact with the ZODB. The scripts have the full component architecture set up when they’re run.
Runners are defined with the appmain decorator:
>>> import logging >>> import gocept.runner >>> work_count = 0 >>> @gocept.runner.appmain(ticks=0.1) ... def worker(): ... import zope.app.appsetup.product ... log = logging.getLogger('test') ... log.info("Working") ... log.info(zope.app.appsetup.product.getProductConfiguration('test')) ... global work_count ... work_count += 1 ... if work_count >= 3: ... return gocept.runner.Exit
The decorated worker takes two arguments now:
The name of an object in the root which will be set as site or None for the root.
The path to a configuration file (zope.conf)
Create a simple zope.conf:
>>> import os.path >>> import tempfile >>> zodb_path = tempfile.mkdtemp() >>> site_zcml = os.path.join( ... os.path.dirname(__file__), 'ftesting.zcml') >>> fd, zope_conf = tempfile.mkstemp() >>> zope_conf_file = os.fdopen(fd, 'w') >>> zope_conf_file.write('''\ ... site-definition %s ... <zodb> ... <filestorage> ... path %s/Data.fs ... </filestorage> ... </zodb> ... <product-config test> ... foo bar ... </product-config> ... <accesslog> ... <logfile> ... path STDOUT ... </logfile> ... </accesslog> ... <eventlog> ... <logfile> ... formatter zope.exceptions.log.Formatter ... path STDOUT ... </logfile> ... </eventlog> ... ''' % (site_zcml, zodb_path)) >>> zope_conf_file.close()
So call the worker:
>>> worker(None, zope_conf) ------ ... INFO test Working ------ ... INFO test {'foo': 'bar'} ------ ... INFO test Working ------ ... INFO test {'foo': 'bar'} ------ ... INFO test Working ------ ... INFO test {'foo': 'bar'}
Signals
The worker-procss can be terminated by SIGTERM and SIGHUP in a sane way. Write a script to a temporary file:
>>> import sys >>> runner_path = os.path.abspath( ... os.path.join(os.path.dirname(__file__), '..', '..')) >>> fd, script_name = tempfile.mkstemp(suffix='.py') >>> exchange_fd, exchange_file_name = tempfile.mkstemp() >>> script = os.fdopen(fd, 'w') >>> script.write("""\ ... import sys ... sys.path[0:0] = %s ... sys.path.insert(0, '%s') ... import gocept.runner ... ... f = open('%s', 'w') ... ... @gocept.runner.appmain(ticks=0.1) ... def worker(): ... f.write("Working.\\n") ... f.flush() ... ... worker(None, '%s') ... """ % (sys.path, runner_path, exchange_file_name, zope_conf)) >>> script.close()
Call the script and wait for it to produce some output:
>>> import signal >>> import subprocess >>> import time >>> exchange = os.fdopen(exchange_fd, 'r+') >>> proc = subprocess.Popen( ... [sys.executable, script_name], ... stdout=subprocess.PIPE) >>> while not exchange.read(): ... time.sleep(0.1) ... exchange.seek(0, 0) >>> exchange.seek(0, 0) >>> print exchange.read(), Working.
Okay, now kill it:
>>> os.kill(proc.pid, signal.SIGTERM)
Wait for the process to really finish and get the output. The runner logs that it was terminated:
>>> stdout, stderr = proc.communicate() >>> print stdout, ------ ... INFO gocept.runner.runner Received signal 15, terminating.
This also works with SIGHUP:
>>> exchange.truncate(0) >>> proc = subprocess.Popen( ... [sys.executable, script_name], ... stdout=subprocess.PIPE) >>> while not exchange.read(): ... time.sleep(0.1) ... exchange.seek(0, 0) >>> exchange.seek(0, 0) >>> print exchange.read(), Working.
Okay, now kill it:
>>> os.kill(proc.pid, signal.SIGHUP) >>> stdout, stderr = proc.communicate() >>> print stdout, ------ ... INFO gocept.runner.runner Received signal 1, terminating.
Clean up:
>>> os.remove(script_name) >>> os.remove(exchange_file_name)
Setting the principal
It is also prossible to create a main loop which runs in an interaction:
>>> import zope.security.management >>> work_count = 0 >>> @gocept.runner.appmain(ticks=0.1, principal='zope.mgr') ... def worker(): ... global work_count ... work_count += 1 ... if work_count >= 3: ... raise SystemExit(1) ... log = logging.getLogger('test') ... interaction = zope.security.management.getInteraction() ... principal = interaction.participations[0].principal ... log.info("Working as %s" % principal.id)
Create a zope.conf which doesn’t set up a logger for stdout but writes to a tmpfile. This is necessary as we’re running in the same process here:
>>> zope_conf_file = open(zope_conf, 'w') >>> zope_conf_file.write('''\ ... site-definition %s ... <zodb> ... <filestorage> ... path %s/Data.fs ... </filestorage> ... </zodb> ... <product-config test> ... foo bar ... </product-config> ... <accesslog> ... </accesslog> ... <eventlog> ... </eventlog> ... ''' % (site_zcml, zodb_path)) >>> zope_conf_file.close()
Call the worker now:
>>> worker(None, zope_conf) ------ ... INFO test Working as zope.mgr ------ ... INFO test Working as zope.mgr
After the worker is run there is no interaction:
>>> zope.security.management.queryInteraction() is None True
Clean up:
>>> import shutil >>> shutil.rmtree(zodb_path) >>> os.remove(zope_conf)
Run once commands
It is often needed to run a command once (or via cron) with the full component architecture. Usually zopectl run is used for this.
>>> import logging >>> import gocept.runner >>> import zope.app.component.hooks >>> @gocept.runner.once() ... def worker(): ... log = logging.getLogger('test') ... log.info("Working.") ... site = zope.app.component.hooks.getSite() ... if hasattr(site, 'store'): ... log.info("Having attribute store.") ... site.store = True
Define a Zope environment:
>>> import os.path >>> import tempfile >>> zodb_path = tempfile.mkdtemp() >>> site_zcml = os.path.join( ... os.path.dirname(__file__), 'ftesting.zcml') >>> fd, zope_conf = tempfile.mkstemp() >>> zope_conf_file = os.fdopen(fd, 'w') >>> zope_conf_file.write('''\ ... site-definition %s ... <zodb> ... <filestorage> ... path %s/Data.fs ... </filestorage> ... </zodb> ... <product-config test> ... foo bar ... </product-config> ... <accesslog> ... <logfile> ... path STDOUT ... </logfile> ... </accesslog> ... <eventlog> ... <logfile> ... formatter zope.exceptions.log.Formatter ... path STDOUT ... </logfile> ... </eventlog> ... ''' % (site_zcml, zodb_path)) >>> zope_conf_file.close()
So call the worker for the first time. It will be terminated after one call:
>>> worker(None, zope_conf) ------ ... INFO test Working.
Calling it a second time indicates that a property was changed in the first run:
>>> worker(None, zope_conf) ------ ... INFO test Working. ------ ... INFO test Having attribute store. ...
Runner details
Main loop
The main loop loops until it encounters a KeyboardInterrupt or a SystemExit exception and calls the worker once a second.
Define a worker function which exits when it is called the 3rd time:
>>> work_count = 0 >>> def worker(): ... print "Working" ... global work_count ... work_count += 1 ... if work_count >= 3: ... raise SystemExit(1)
Call the main loop:
>>> import gocept.runner.runner >>> gocept.runner.runner.MainLoop(getRootFolder(), 0.1, worker)() Working Working Working >>> work_count 3
During the loop the site is set:
>>> import zope.app.component.hooks >>> zope.app.component.hooks.getSite() is None True >>> def worker(): ... print zope.app.component.hooks.getSite() ... raise SystemExit(1) >>> gocept.runner.runner.MainLoop(getRootFolder(), 0.1, worker)() <zope.site.folder.Folder object at 0x...>
After the loop, no site is set again:
>>> zope.app.component.hooks.getSite() is None True
When the worker passes without error a transaction is commited:
>>> work_count = 0 >>> def worker(): ... print "Working" ... global work_count ... work_count += 1 ... if work_count >= 2: ... raise SystemExit(1) ... site = zope.app.component.hooks.getSite() ... site.worker_done = 1 >>> gocept.runner.runner.MainLoop(getRootFolder(), 0.1, worker)() Working Working
We have set the attribute worker_done now:
>>> getRootFolder().worker_done 1
When the worker produces an error, the transaction is aborted:
>>> work_count = 0 >>> def worker(): ... global work_count ... work_count += 1 ... print "Working" ... site = zope.app.component.hooks.getSite() ... site.worker_done += 1 ... if work_count < 3: ... raise ValueError('hurz') ... raise SystemExit(1) >>> gocept.runner.runner.MainLoop(getRootFolder(), 0.1, worker)() Working Working Working
We still have the attribute worker_done set to 1:b
>>> getRootFolder().worker_done 1
Controlling sleep time
The worker function can control the sleep time[#log-handler]_
>>> work_count = 0 >>> def worker(): ... global work_count ... work_count += 1 ... new_sleep = work_count * 0.1 ... if work_count == 3: ... print "Will sleep default" ... return None ... if work_count > 3: ... raise SystemExit(1) ... print "Will sleep", new_sleep ... return new_sleep >>> gocept.runner.runner.MainLoop(getRootFolder(), 0.15, worker)() Will sleep 0.1 Will sleep 0.2 Will sleep default
The real sleep values are in the log:
>>> print log.getvalue(), new transaction commit Sleeping 0.1 seconds new transaction commit Sleeping 0.2 seconds new transaction commit Sleeping 0.15 seconds new transaction abort
Restore old log handler:
>>> logging.root.removeHandler(log_handler) >>> logging.root.setLevel(old_log_level)
Changes
0.3 (2009-04-15)
When a worker fails the default sleep time (instead of the last one) will be used.
0.2 (2009-04-09)
Added a clean way to exit the runner (by returning gocept.runner.Exit).
0.1 (2009-04-07)
first public release
Project details
Release history Release notifications | RSS feed
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.