[views] implement json / jsonp export views (closes #1942658)

Json export views are based on the same model as CSV export views. There are two distinct views :

  • jsonexport : direct conversion of the result set into json
  • ejsonexport : convert entities into json

The JSONP parameter is named callback (same as on geonames, dbepdia and a lot of sites)

An optional _indent request parameter can be passed to pretty print the results.

authorAdrien Di Mascio <Adrien.DiMascio@logilab.fr>
changesetdf15d194a134
branchdefault
phasepublic
hiddenno
parent revision#65e460690139 [form, entity] refactor '__linkto', now handled by the entity form, not the entity itself. Closes #1931543
child revision#662ad647306f [json] restore 2.5 compat
files modified by this revision
dbapi.py
doc/book/en/devweb/controllers.rst
entity.py
utils.py
web/application.py
web/test/unittest_application.py
web/test/unittest_views_json.py
web/views/json.py
# HG changeset patch
# User Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
# Date 1317142031 -7200
# Tue Sep 27 18:47:11 2011 +0200
# Node ID df15d194a134b736d442e0df107d9b0e2410063e
# Parent 65e460690139ace95abdcc0ed57bd50390a4bd21
[views] implement json / jsonp export views (closes #1942658)

Json export views are based on the same model as CSV export views.
There are two distinct views :

- *jsonexport* : direct conversion of the result set into json
- *ejsonexport* : convert entities into json

The JSONP parameter is named ``callback`` (same as on geonames, dbepdia
and a lot of sites)

An optional `_indent` request parameter can be passed to pretty print
the results.

diff --git a/dbapi.py b/dbapi.py
@@ -221,17 +221,36 @@
1      additionel credential might be required"""
2      cnxprops = ConnectionProperties('inmemory')
3      return repo_connect(repo, login, cnxprops=cnxprops, **kwargs)
4 
5  def in_memory_repo_cnx(config, login, **kwargs):
6 -    """usefull method for testing and scripting to get a dbapi.Connection
7 +    """useful method for testing and scripting to get a dbapi.Connection
8      object connected to an in-memory repository instance
9      """
10      # connection to the CubicWeb repository
11      repo = in_memory_repo(config)
12      return repo, in_memory_cnx(repo, login, **kwargs)
13 
14 +
15 +def anonymous_session(vreg):
16 +    """return a new anonymous session
17 +
18 +    raises an AuthenticationError if anonymous usage is not allowed
19 +    """
20 +    anoninfo = vreg.config.anonymous_user()
21 +    if anoninfo is None: # no anonymous user
22 +        raise AuthenticationError('anonymous access is not authorized')
23 +    anon_login, anon_password = anoninfo
24 +    cnxprops = ConnectionProperties(vreg.config.repo_method)
25 +    # use vreg's repository cache
26 +    repo = vreg.config.repository(vreg)
27 +    anon_cnx = repo_connect(repo, anon_login,
28 +                            cnxprops=cnxprops, password=anon_password)
29 +    anon_cnx.vreg = vreg
30 +    return DBAPISession(anon_cnx, anon_login)
31 +
32 +
33  class _NeedAuthAccessMock(object):
34      def __getattribute__(self, attr):
35          raise AuthenticationError()
36      def __nonzero__(self):
37          return False
diff --git a/doc/book/en/devweb/controllers.rst b/doc/book/en/devweb/controllers.rst
@@ -24,13 +24,27 @@
38 
39  * the JSon controller (same module) provides services for Ajax calls,
40    typically using JSON as a serialization format for input, and
41    sometimes using either JSON or XML for output;
42 
43 +* the JSonpController is a wrapper around the ``ViewController`` that
44 +  provides jsonp_ services. Padding can be specified with the
45 +  ``callback`` request parameter. Only *jsonexport* / *ejsonexport*
46 +  views can be used. If another ``vid`` is specified, it will be
47 +  ignored and replaced by *jsonexport*. Request is anonymized
48 +  to avoid returning sensitive data and reduce the risks of CSRF attacks;
49 +
50  * the Login/Logout controllers make effective user login or logout
51    requests
52 
53 +.. warning::
54 +
55 +  JsonController will probably be renamed into AjaxController soon since
56 +  it has nothing to do with json per se.
57 +
58 +.. _jsonp: http://en.wikipedia.org/wiki/JSONP
59 +
60  `Edition`:
61 
62  * the Edit controller (see :ref:`edit_controller`) handles CRUD
63    operations in response to a form being submitted; it works in close
64    association with the Forms, to which it delegates some of the work
diff --git a/entity.py b/entity.py
@@ -455,11 +455,11 @@
65 
66      def __json_encode__(self):
67          """custom json dumps hook to dump the entity's eid
68          which is not part of dict structure itself
69          """
70 -        dumpable = dict(self)
71 +        dumpable = self.cw_attr_cache.copy()
72          dumpable['eid'] = self.eid
73          return dumpable
74 
75      def cw_adapt_to(self, interface):
76          """return an adapter the entity to the given interface name.
diff --git a/utils.py b/utils.py
@@ -477,14 +477,12 @@
77 
78      class CubicWebJsonEncoder(json.JSONEncoder):
79          """define a json encoder to be able to encode yams std types"""
80 
81          def default(self, obj):
82 -            if hasattr(obj, 'eid'):
83 -                d = obj.cw_attr_cache.copy()
84 -                d['eid'] = obj.eid
85 -                return d
86 +            if hasattr(obj, '__json_encode__'):
87 +                return obj.__json_encode__()
88              if isinstance(obj, datetime.datetime):
89                  return ustrftime(obj, '%Y/%m/%d %H:%M:%S')
90              elif isinstance(obj, datetime.date):
91                  return ustrftime(obj, '%Y/%m/%d')
92              elif isinstance(obj, datetime.time):
@@ -498,12 +496,12 @@
93              except TypeError:
94                  # we never ever want to fail because of an unknown type,
95                  # just return None in those cases.
96                  return None
97 
98 -    def json_dumps(value):
99 -        return json.dumps(value, cls=CubicWebJsonEncoder)
100 +    def json_dumps(value, **kwargs):
101 +        return json.dumps(value, cls=CubicWebJsonEncoder, **kwargs)
102 
103 
104      class JSString(str):
105          """use this string sub class in values given to :func:`js_dumps` to
106          insert raw javascript chain in some JSON string
diff --git a/web/application.py b/web/application.py
@@ -21,29 +21,40 @@
107 
108  __docformat__ = "restructuredtext en"
109 
110  import sys
111  from time import clock, time
112 +from contextlib import contextmanager
113 
114  from logilab.common.deprecation import deprecated
115 
116  from rql import BadRQLQuery
117 
118  from cubicweb import set_log_methods, cwvreg
119  from cubicweb import (
120      ValidationError, Unauthorized, AuthenticationError, NoSelectableObject,
121      BadConnectionId, CW_EVENT_MANAGER)
122 -from cubicweb.dbapi import DBAPISession
123 +from cubicweb.dbapi import DBAPISession, anonymous_session
124  from cubicweb.web import LOGGER, component
125  from cubicweb.web import (
126      StatusResponse, DirectResponse, Redirect, NotFound, LogOut,
127      RemoteCallFailed, InvalidSession, RequestError)
128 
129  # make session manager available through a global variable so the debug view can
130  # print information about web session
131  SESSION_MANAGER = None
132 
133 +
134 +@contextmanager
135 +def anonymized_request(req):
136 +    orig_session = req.session
137 +    req.set_session(anonymous_session(req.vreg))
138 +    try:
139 +        yield req
140 +    finally:
141 +        req.set_session(orig_session)
142 +
143  class AbstractSessionManager(component.Component):
144      """manage session data associated to a session identifier"""
145      __regid__ = 'sessionmanager'
146 
147      def __init__(self, vreg):
diff --git a/web/test/unittest_application.py b/web/test/unittest_application.py
@@ -29,10 +29,11 @@
148  from cubicweb import AuthenticationError, Unauthorized
149  from cubicweb.devtools.testlib import CubicWebTC
150  from cubicweb.devtools.fake import FakeRequest
151  from cubicweb.web import LogOut, Redirect, INTERNAL_FIELD_VALUE
152  from cubicweb.web.views.basecontrollers import ViewController
153 +from cubicweb.web.application import anonymized_request
154 
155  class FakeMapping:
156      """emulates a mapping module"""
157      def __init__(self):
158          self.ENTITIES_MAP = {}
@@ -422,10 +423,22 @@
159          req.form['__password'] = self.admpassword
160          self.assertAuthSuccess(req, origsession)
161          self.assertRaises(LogOut, self.app_publish, req, 'logout')
162          self.assertEqual(len(self.open_sessions), 0)
163 
164 +    def test_anonymized_request(self):
165 +        req = self.request()
166 +        self.assertEqual(req.session.login, self.admlogin)
167 +        # admin should see anon + admin
168 +        self.assertEqual(len(list(req.find_entities('CWUser'))), 2)
169 +        with anonymized_request(req):
170 +            self.assertEqual(req.session.login, 'anon')
171 +            # anon should only see anon user
172 +            self.assertEqual(len(list(req.find_entities('CWUser'))), 1)
173 +        self.assertEqual(req.session.login, self.admlogin)
174 +        self.assertEqual(len(list(req.find_entities('CWUser'))), 2)
175 +
176      def test_non_regr_optional_first_var(self):
177          req = self.request()
178          # expect a rset with None in [0][0]
179          req.form['rql'] = 'rql:Any OV1, X WHERE X custom_workflow OV1?'
180          self.app_publish(req)
diff --git a/web/test/unittest_views_json.py b/web/test/unittest_views_json.py
@@ -0,0 +1,46 @@
181 +from cubicweb.devtools.testlib import CubicWebTC
182 +
183 +from json import loads
184 +
185 +class JsonViewsTC(CubicWebTC):
186 +
187 +    def test_json_rsetexport(self):
188 +        req = self.request()
189 +        rset = req.execute('Any GN,COUNT(X) GROUPBY GN ORDERBY GN WHERE X in_group G, G name GN')
190 +        data = self.view('jsonexport', rset)
191 +        self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/json'])
192 +        self.assertEqual(data, '[["guests", 1], ["managers", 1]]')
193 +
194 +    def test_json_rsetexport_with_jsonp(self):
195 +        req = self.request()
196 +        req.form.update({'callback': 'foo',
197 +                         'rql': 'Any GN,COUNT(X) GROUPBY GN ORDERBY GN WHERE X in_group G, G name GN',
198 +                         })
199 +        data = self.ctrl_publish(req, ctrl='jsonp')
200 +        self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/javascript'])
201 +        # because jsonp anonymizes data, only 'guests' group should be found
202 +        self.assertEqual(data, 'foo([["guests", 1]])')
203 +
204 +    def test_json_rsetexport_with_jsonp_and_bad_vid(self):
205 +        req = self.request()
206 +        req.form.update({'callback': 'foo',
207 +                         'vid': 'table', # <-- this parameter should be ignored by jsonp controller
208 +                         'rql': 'Any GN,COUNT(X) GROUPBY GN ORDERBY GN WHERE X in_group G, G name GN',
209 +                         })
210 +        data = self.ctrl_publish(req, ctrl='jsonp')
211 +        self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/javascript'])
212 +        # result should be plain json, not the table view
213 +        self.assertEqual(data, 'foo([["guests", 1]])')
214 +
215 +    def test_json_ersetexport(self):
216 +        req = self.request()
217 +        rset = req.execute('Any G ORDERBY GN WHERE G is CWGroup, G name GN')
218 +        data = loads(self.view('ejsonexport', rset))
219 +        self.assertEqual(req.headers_out.getRawHeaders('content-type'), ['application/json'])
220 +        self.assertEqual(data[0]['name'], 'guests')
221 +        self.assertEqual(data[1]['name'], 'managers')
222 +
223 +
224 +if __name__ == '__main__':
225 +    from logilab.common.testlib import unittest_main
226 +    unittest_main()
diff --git a/web/views/json.py b/web/views/json.py
@@ -0,0 +1,112 @@
227 +# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
228 +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
229 +#
230 +# This file is part of CubicWeb.
231 +#
232 +# CubicWeb is free software: you can redistribute it and/or modify it under the
233 +# terms of the GNU Lesser General Public License as published by the Free
234 +# Software Foundation, either version 2.1 of the License, or (at your option)
235 +# any later version.
236 +#
237 +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
238 +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
239 +# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
240 +# details.
241 +#
242 +# You should have received a copy of the GNU Lesser General Public License along
243 +# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
244 +"""json export views"""
245 +
246 +__docformat__ = "restructuredtext en"
247 +_ = unicode
248 +
249 +from cubicweb.utils import json_dumps
250 +from cubicweb.view import EntityView, AnyRsetView
251 +from cubicweb.web.application import anonymized_request
252 +from cubicweb.web.views import basecontrollers
253 +
254 +class JsonpController(basecontrollers.ViewController):
255 +    """The jsonp controller is the same as a ViewController but :
256 +
257 +    - anonymize request (avoid CSRF attacks)
258 +    - if ``vid`` parameter is passed, make sure it's sensible (i.e. either
259 +      "jsonexport" or "ejsonexport")
260 +    - if ``callback`` request parameter is passed, it's used as json padding
261 +
262 +
263 +    Response's content-type will either be ``application/javascript`` or
264 +    ``application/json`` depending on ``callback`` parameter presence or not.
265 +    """
266 +    __regid__ = 'jsonp'
267 +
268 +    def publish(self, rset=None):
269 +        if 'vid' in self._cw.form:
270 +            vid = self._cw.form['vid']
271 +            if vid not in ('jsonexport', 'ejsonexport'):
272 +                self.warning("vid %s can't be used with jsonp controller, "
273 +                             "falling back to jsonexport", vid)
274 +                self._cw.form['vid'] = 'jsonexport'
275 +        else: # if no vid is specified, use jsonexport
276 +            self._cw.form['vid'] = 'jsonexport'
277 +        with anonymized_request(self._cw):
278 +            json_data = super(JsonpController, self).publish(rset)
279 +            if 'callback' in self._cw.form: # jsonp
280 +                json_padding = self._cw.form['callback']
281 +                # use ``application/javascript`` is ``callback`` parameter is
282 +                # provided, let ``application/json`` otherwise
283 +                self._cw.set_content_type('application/javascript')
284 +                json_data = '%s(%s)' % (json_padding, json_data)
285 +        return json_data
286 +
287 +
288 +class JsonMixIn(object):
289 +    """mixin class for json views
290 +
291 +    Handles the following optional request parameters:
292 +
293 +    - ``_indent`` : must be an integer. If found, it is used to pretty print
294 +      json output
295 +    """
296 +    templatable = False
297 +    content_type = 'application/json'
298 +    binary = True
299 +
300 +    def wdata(self, data):
301 +        if '_indent' in self._cw.form:
302 +            indent = int(self._cw.form['_indent'])
303 +        else:
304 +            indent = None
305 +        self.w(json_dumps(data, indent=indent))
306 +
307 +
308 +class JsonRsetView(JsonMixIn, AnyRsetView):
309 +    """dumps raw result set in JSON format"""
310 +    __regid__ = 'jsonexport'
311 +    title = _('json-export-view')
312 +
313 +    def call(self):
314 +        # XXX mimic w3c recommandations to serialize SPARQL results in json ?
315 +        #     http://www.w3.org/TR/rdf-sparql-json-res/
316 +        self.wdata(self.cw_rset.rows)
317 +
318 +
319 +class JsonEntityView(JsonMixIn, EntityView):
320 +    """dumps rset entities in JSON
321 +
322 +    The following additional metadata is added to each row :
323 +
324 +    - ``__cwetype__`` : entity type
325 +    """
326 +    __regid__ = 'ejsonexport'
327 +    title = _('json-entities-export-view')
328 +
329 +    def call(self):
330 +        entities = []
331 +        for entity in self.cw_rset.entities():
332 +            entity.complete() # fetch all attributes
333 +            # hack to add extra metadata
334 +            entity.cw_attr_cache.update({
335 +                    '__cwetype__': entity.__regid__,
336 +                    })
337 +            entities.append(entity)
338 +        self.wdata(entities)