[dbapi] add possibility to connect to a remote ZMQRepository (closes #2290126)

authorVincent Michel <vincent.michel@logilab.fr>
changeset0e3b41118631
branchdefault
phasepublic
hiddenno
parent revision#02f4f01375e8 [repository] fire 'server_shutdown' hooks before waiting for theads
child revision#c1cc2f1cd177 [zmq] add unit tests for ZMQ-based repository (server and dbapi)
files modified by this revision
cwctl.py
dbapi.py
misc/migration/3.15.0_common.py
zmqclient.py
# HG changeset patch
# User Vincent Michel <vincent.michel@logilab.fr>
# Date 1334070544 -7200
# Tue Apr 10 17:09:04 2012 +0200
# Node ID 0e3b41118631d9788c4725595b80b5cd19572a81
# Parent 02f4f01375e894a8f9f7df0cb44a9fea4fca091b
[dbapi] add possibility to connect to a remote ZMQRepository (closes #2290126)

diff --git a/cwctl.py b/cwctl.py
@@ -1,6 +1,6 @@
1 -# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
2 +# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
4  #
5  # This file is part of CubicWeb.
6  #
7  # CubicWeb is free software: you can redistribute it and/or modify it under the
@@ -36,10 +36,12 @@
8          """win32 getpgid implementation"""
9 
10 
11  from os.path import exists, join, isfile, isdir, dirname, abspath
12 
13 +from urlparse import urlparse
14 +
15  from logilab.common.clcommands import CommandLine
16  from logilab.common.shellutils import ASK
17 
18  from cubicweb import ConfigurationError, ExecutionError, BadCommandUsage
19  from cubicweb.utils import support_args
@@ -865,35 +867,49 @@
20           {'short': 'f', 'action' : 'store_true',
21            'help': 'don\'t check instance is up to date.',
22            'group': 'local'
23            }),
24 
25 -        ('pyro',
26 -         {'short': 'P', 'action' : 'store_true',
27 -          'help': 'connect to a running instance through Pyro.',
28 -          'group': 'remote',
29 -          }),
30 -        ('pyro-ns-host',
31 -         {'short': 'H', 'type' : 'string', 'metavar': '<host[:port]>',
32 -          'help': 'Pyro name server host. If not set, will be detected by '
33 -          'using a broadcast query.',
34 +        ('repo-uri',
35 +         {'short': 'H', 'type' : 'string', 'metavar': '<protocol>://<[host][:port]>',
36 +          'help': 'URI of the CubicWeb repository to connect to. URI can be \
37 +pyro://[host:port] the Pyro name server host; if the pyro nameserver is not set, \
38 +it will be detected by using a broadcast query, a ZMQ URL or \
39 +inmemory:// (default) use an in-memory repository.',
40            'group': 'remote'
41            }),
42          )
43 
44      def run(self, args):
45          appid = args.pop(0)
46 -        if self.config.pyro:
47 +        if self.config.repo_uri:
48 +            uri = urlparse(self.config.repo_uri)
49 +            if uri.scheme == 'pyro':
50 +                cnxtype = uri.scheme
51 +                hostport = uri.netloc
52 +            elif uri.scheme == 'inmemory':
53 +                cnxtype = ''
54 +                hostport = ''
55 +            else:
56 +                cnxtype = 'zmq'
57 +                hostport = self.config.repo_uri
58 +        else:
59 +            cnxtype = ''
60 +
61 +        if cnxtype:
62              from cubicweb import AuthenticationError
63 -            from cubicweb.dbapi import connect
64 +            from cubicweb.dbapi import connect, ConnectionProperties
65              from cubicweb.server.utils import manager_userpasswd
66              from cubicweb.server.migractions import ServerMigrationHelper
67 +            cnxprops = ConnectionProperties(cnxtype=cnxtype)
68 +
69              while True:
70                  try:
71                      login, pwd = manager_userpasswd(msg=None)
72                      cnx = connect(appid, login=login, password=pwd,
73 -                                  host=self.config.pyro_ns_host, mulcnx=False)
74 +                                  host=hostport, mulcnx=False,
75 +                                  cnxprops=cnxprops)
76                  except AuthenticationError, ex:
77                      print ex
78                  except (KeyboardInterrupt, EOFError):
79                      print
80                      sys.exit(0)
@@ -925,11 +941,11 @@
81                      mih.cmd_process_script(script, scriptargs=args)
82                      mih.commit()
83              else:
84                  mih.interactive_shell()
85          finally:
86 -            if not self.config.pyro:
87 +            if not cnxtype: # shutdown in-memory repo
88                  mih.shutdown()
89              else:
90                  cnx.close()
91 
92 
diff --git a/dbapi.py b/dbapi.py
@@ -91,20 +91,22 @@
93      """get a proxy object to the CubicWeb repository, using a specific RPC method.
94 
95      Only 'in-memory' and 'pyro' are supported for now. Either vreg or config
96      argument should be given
97      """
98 -    assert method in ('pyro', 'inmemory')
99 +    assert method in ('pyro', 'inmemory', 'zmq')
100      assert vreg or config
101      if vreg and not config:
102          config = vreg.config
103      if method == 'inmemory':
104          # get local access to the repository
105          from cubicweb.server.repository import Repository
106          from cubicweb.server.utils import TasksManager
107          return Repository(config, TasksManager(), vreg=vreg)
108 -
109 +    elif method == 'zmq':
110 +        from cubicweb.zmqclient import ZMQRepositoryClient
111 +        return ZMQRepositoryClient(config, vreg=vreg)
112      else: # method == 'pyro'
113          # resolve the Pyro object
114          from logilab.common.pyro_ext import ns_get_proxy, get_proxy
115          pyroid = database or config['pyro-instance-id'] or config.appid
116          try:
@@ -145,12 +147,12 @@
117 
118      :login:
119        the user login to use to authenticate.
120 
121      :host:
122 -      the pyro nameserver host. Will be detected using broadcast query if
123 -      unspecified.
124 +      - pyro: nameserver host. Will be detected using broadcast query if unspecified
125 +      - zmq: repository host socket address
126 
127      :group:
128        the instance's pyro nameserver group. You don't have to specify it unless
129        tweaked in instance's configuration.
130 
@@ -183,10 +185,12 @@
131          config = cwconfig.CubicWebNoAppConfiguration()
132          if host:
133              config.global_set_option('pyro-ns-host', host)
134          if group:
135              config.global_set_option('pyro-ns-group', group)
136 +    elif method == 'zmq':
137 +        config = cwconfig.CubicWebNoAppConfiguration()
138      else:
139          assert database
140          config = cwconfig.instance_configuration(database)
141      repo = get_repository(method, database, config=config)
142      if method == 'inmemory':
diff --git a/misc/migration/3.15.0_common.py b/misc/migration/3.15.0_common.py
@@ -1,2 +1,5 @@
143  undo_actions = config.cfgfile_parser.get('MAIN', 'undo-support', False)
144  config.global_set_option('undo-enabled', bool(undo_actions))
145 +pyro_actions = config.cfgfile_parser.get('REMOTE', 'pyro', False)
146 +if pyro_actions:
147 +    config.global_set_option('repo-uri', 'pyro://')
diff --git a/zmqclient.py b/zmqclient.py
@@ -0,0 +1,61 @@
148 +# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
149 +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
150 +#
151 +# This file is part of CubicWeb.
152 +#
153 +# CubicWeb is free software: you can redistribute it and/or modify it under the
154 +# terms of the GNU Lesser General Public License as published by the Free
155 +# Software Foundation, either version 2.1 of the License, or (at your option)
156 +# any later version.
157 +#
158 +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
159 +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
160 +# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
161 +# details.
162 +#
163 +# You should have received a copy of the GNU Lesser General Public License along
164 +# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
165 +"""Source to query another RQL repository using pyro"""
166 +
167 +__docformat__ = "restructuredtext en"
168 +_ = unicode
169 +
170 +from functools import partial
171 +import zmq
172 +
173 +
174 +# XXX hack to overpass old zmq limitation that force to have
175 +# only one context per python process
176 +try:
177 +    from cubicweb.server.cwzmq import ctx
178 +except ImportError:
179 +    ctx = zmq.Context()
180 +
181 +class ZMQRepositoryClient(object):
182 +    """
183 +    This class delegate the overall repository stuff to a remote source.
184 +
185 +    So calling a method of this repository will results on calling the
186 +    corresponding method of the remote source repository.
187 +
188 +    Any raised exception on the remote source is propagated locally.
189 +
190 +    ZMQ is used as the transport layer and cPickle is used to serialize data.
191 +    """
192 +
193 +    def __init__(self, config, vreg=None):
194 +        self.config = config
195 +        self.vreg = vreg
196 +        self.socket = ctx.socket(zmq.REQ)
197 +        self.host = config.get('base-url')
198 +        self.socket.connect(self.host)
199 +
200 +    def __zmqcall__(self, name, *args, **kwargs):
201 +         self.socket.send_pyobj([name, args, kwargs])
202 +         result = self.socket.recv_pyobj()
203 +         if isinstance(result, BaseException):
204 +             raise result
205 +         return result
206 +
207 +    def __getattr__(self, name):
208 +        return partial(self.__zmqcall__, name)