[tests] Put loading utilities in a separate module

Related to #6718871

authorChristophe de Vienne <cdevienne@gmail.com>
changeset92e4fbc12db6
branchdefault
phasepublic
hiddenno
parent revision#17163cccf37a Use reregister instead of register
child revision#dc58055d258d [tests] Fully tests loader
files modified by this revision
controller.py
loader.py
test/test_base.py
test/test_loader.py
# HG changeset patch
# User Christophe de Vienne <cdevienne@gmail.com>
# Date 1441650949 -7200
# Mon Sep 07 20:35:49 2015 +0200
# Node ID 92e4fbc12db617ab999eb02ca8c2fcecfc9e5446
# Parent 17163cccf37a6295b45b7feb51798ee47cec1683
[tests] Put loading utilities in a separate module

Related to #6718871

diff --git a/controller.py b/controller.py
@@ -1,10 +1,9 @@
1  """ 'webservice' controller implementation
2  """
3  import inspect
4  import itertools
5 -import re
6  import sys
7 
8  from cubicweb.web.controller import Controller
9  from cubicweb.web import NotFound
10  from cubicweb.predicates import yes
@@ -14,392 +13,21 @@
11  import wsme
12  import wsme.rest.json
13  import wsme.rest.xml
14 
15  from wsme.types import text
16 -from cubes.wsme.types import PassThroughType, JsonData, Any, Base, wsattr
17 +from cubes.wsme.types import PassThroughType, JsonData, Any, Base
18  from cubes.wsme.predicates import match_ws_etype
19 
20 -from rql import nodes
21 -
22 -from rqlquery.filter import FilterParser
23 -
24 -from rqlquery import query
25 +from cubes.wsme.loader import to_fetchtree, load_entities
26 
27  restformats = {
28      'json': wsme.rest.json,
29      'xml': wsme.rest.xml
30  }
31 
32 
33 -def get_fetchable_attributes(user, eschema):
34 -    relations = []
35 -    for rschema in eschema.ordered_relations():
36 -        card = eschema.rdef(rschema.type, takefirst=True).cardinality[0]
37 -        if (card in ('*', '+')
38 -                # cardinality '?' with polymorphic relations raises
39 -                # https://www.cubicweb.org/ticket/4482382
40 -                or card == '?' and len(rschema.objects(eschema)) > 1):
41 -            continue
42 -
43 -        if not rschema.final:
44 -            continue
45 -        if rschema.type in (
46 -                'eid', 'has_text', 'cw_source', 'is'):
47 -            continue
48 -        if rschema.objects()[0].type in ('Password',):
49 -            continue
50 -        rdef = eschema.rdef(rschema)
51 -        if not user.matching_groups(rdef.get_groups('read')):
52 -            continue
53 -        relations.append(rschema)
54 -    return relations
55 -
56 -
57 -def get_fetchable_unary_relations(user, eschema):
58 -    relations = []
59 -    for rschema in eschema.ordered_relations():
60 -        card = eschema.rdef(rschema.type, takefirst=True).cardinality[0]
61 -        if (card in ('*', '+')
62 -                # cardinality '?' with polymorphic relations raises
63 -                # https://www.cubicweb.org/ticket/4482382
64 -                or card == '?' and len(rschema.objects(eschema)) > 1):
65 -            continue
66 -
67 -        if rschema.final:
68 -            continue
69 -
70 -        if rschema.type in (
71 -                'eid', 'has_text', 'cw_source', 'is'):
72 -            continue
73 -        if any(
74 -                not user.matching_groups(es.get_groups('read'))
75 -                for es in rschema.objects(eschema)):
76 -            continue
77 -        relations.append(rschema)
78 -    return relations
79 -
80 -
81 -def get_unfetchable_unary_relations(user, eschema):
82 -    relations = []
83 -
84 -    for rschema in eschema.ordered_relations():
85 -        card = eschema.rdef(rschema.type, takefirst=True).cardinality[0]
86 -        if card == '?' and len(rschema.objects(eschema)) > 1:
87 -            relations.append(rschema)
88 -
89 -    return relations
90 -
91 -
92 -def tree_touch_path(tree, path):
93 -    node = tree
94 -    for name in path:
95 -        node = node.setdefault(name, {})
96 -
97 -
98 -def to_fetchtree(fetchlist, keyonly=False):
99 -    if fetchlist is None:
100 -        if keyonly:
101 -            return {}
102 -        else:
103 -            return {'': {}}
104 -
105 -    fetchtree = {}
106 -
107 -    for rel in fetchlist:
108 -        tree_touch_path(fetchtree, rel.split('.'))
109 -
110 -    return fetchtree
111 -
112 -
113 -def get_columns(cnx, cwetype, fetchtree):
114 -    """ Returns columns to query for filling cwetype
115 -
116 -    Recursively calls itself for inlined relations
117 -
118 -    Returns
119 -    - a mapping for itself and the inlined relations (recursively)
120 -
121 -    """
122 -    mapping = {'eid': None}
123 -
124 -    if fetchtree in (None, {}) or cwetype.__etype__ == 'Any':
125 -        mapping['modification_date'] = None
126 -
127 -    else:
128 -        eschema = cnx.vreg.schema.eschema(cwetype.__etype__)
129 -
130 -        for rtype in get_fetchable_attributes(cnx.user, eschema):
131 -            mapping[rtype.type] = None
132 -
133 -        for rtype in get_fetchable_unary_relations(cnx.user, eschema):
134 -            if not hasattr(cwetype, rtype.type):
135 -                continue
136 -
137 -            tgt_cwetype = getattr(cwetype, rtype.type).datatype
138 -
139 -            mapping[rtype.type] = get_columns(
140 -                cnx, tgt_cwetype, fetchtree.get(rtype.type))
141 -
142 -    return mapping
143 -
144 -
145 -def add_columns(q, mapping, col_counter, prefix=()):
146 -    """ Add columns of the mapping to the query
147 -
148 -    Returns the updated query
149 -    """
150 -    mapping['eid'] = col_counter.next()
151 -
152 -    for key in list(mapping):
153 -        value = mapping[key]
154 -
155 -        if value is None:
156 -            q = q.add_column(prefix + (key,))
157 -            mapping[key] = col_counter.next()
158 -
159 -        elif isinstance(value, dict):
160 -            q = q.add_column(prefix + (key,))
161 -
162 -            q = add_columns(q, value, col_counter, prefix + (key,))
163 -
164 -    return q
165 -
166 -_rtype_re = re.compile(
167 -    r'(?P<rev>\<)?(?P<rtype>[^.[]+)(\[(?P<etypes>[^\]]+)\])?')
168 -
169 -
170 -def get_unloaded_relations(cnx, cwetype, mapping, fetchtree):
171 -    # for each key in fetchtree not present in mapping, check the actual
172 -    # relation type (final or not) and returns it.
173 -
174 -    # for each key in both trees that is a dict with content
175 -    # do a recursive call
176 -
177 -    schema = cnx.vreg.schema
178 -
179 -    relations = []
180 -
181 -    if cwetype.__etype__ == 'Any':
182 -        return ()
183 -
184 -    eschema = schema.eschema(cwetype.__etype__)
185 -    # automatically add the ?* relations because they are expected by the
186 -    # clients but not automatically loaded (see get_fetchable_unary_relations)
187 -    for rschema in get_unfetchable_unary_relations(cnx.user, eschema):
188 -        # make sure the relation exists as an attribute:
189 -        attr = filter(
190 -            lambda a: (
191 -                isinstance(a, wsattr)
192 -                and a.rtype == rschema.type
193 -                and a.role == 'subject'),
194 -            cwetype._wsme_attributes)
195 -        if attr:
196 -            attr = attr[0]
197 -            if attr.key not in fetchtree and attr.key not in mapping:
198 -                relations.append(((attr.key,), attr, None))
199 -
200 -    for key in fetchtree:
201 -        if key in mapping and not(fetchtree[key]):
202 -            continue
203 -
204 -        if key in ('', '*'):
205 -            continue
206 -
207 -        m = _rtype_re.match(key)
208 -        if m is None:
209 -            raise ValueError("Invalid rtype in fetchtree: {}".format(key))
210 -
211 -        name = (m.group('rev') or '') + m.group('rtype')
212 -
213 -        attr = cwetype.attr_by_name(name)
214 -
215 -        if not isinstance(attr, wsattr):
216 -            continue
217 -
218 -        rtype = attr.rtype
219 -
220 -        rschema = schema.rschema(rtype)
221 -
222 -        if rschema.final:
223 -            # TODO handle fetching specific attributes
224 -            # (not in this function obviously)
225 -            continue
226 -
227 -        if key not in mapping:
228 -            relations.append(((key,), attr, fetchtree[key]))
229 -        else:
230 -            relations.extend((
231 -                ((key,) + path, t, f)
232 -                for path, t, f in get_unloaded_relations(
233 -                    cnx,
234 -                    attr.datatype.item_type
235 -                    if wsme.types.isarray(attr.datatype)
236 -                    else attr.datatype,
237 -                    mapping[key],
238 -                    fetchtree[key])
239 -            ))
240 -
241 -    return relations
242 -
243 -
244 -def make_iter(obj_or_iter):
245 -    if obj_or_iter is None:
246 -        return iter(())
247 -    if isinstance(obj_or_iter, (list, dict, set, tuple)):
248 -        return iter(obj_or_iter)
249 -    return iter((obj_or_iter,))
250 -
251 -
252 -def get_entities(roots, path):
253 -    entities = roots
254 -
255 -    for attrname in path:
256 -        if not entities:
257 -            break
258 -        key = entities[0].attr_by_name(attrname).key
259 -        entities = list(itertools.chain(
260 -            *[make_iter(getattr(o, key)) for o in entities]))
261 -
262 -    return entities
263 -
264 -
265 -def load_entities(
266 -        cnx, cwetype,
267 -        orderby=None, query_filter=None, limit=None, offset=None,
268 -        fetchtree=None):
269 -
270 -    q = query.Query(cnx.vreg.schema, cwetype.__etype__)
271 -
272 -    mapping = get_columns(cnx, cwetype, fetchtree)
273 -
274 -    col_count = iter(xrange(999))
275 -
276 -    q = add_columns(q, mapping, col_count)
277 -
278 -    if orderby:
279 -        q = q.orderby(*orderby)
280 -    if query_filter:
281 -        q = q.filter(FilterParser(
282 -            cnx.vreg.schema, cwetype.__etype__, query_filter
283 -        ).parse())
284 -    if limit:
285 -        q = q.limit(limit)
286 -    if offset:
287 -        q = q.offset(offset)
288 -
289 -    rset = q.execute(cnx)
290 -
291 -    entities = [
292 -        cwetype.from_row(cnx, rset, row_i, row, mapping, fetchtree)
293 -        for row_i, row in enumerate(rset)
294 -    ]
295 -
296 -    # nested relation loading
297 -    # scan the mapping and get all unloaded relations that are in fetchtree.
298 -    # Load them, complete the mapping (which is now only a tree of what was
299 -    # loaded), and redo until every relation is loaded
300 -
301 -    while True:
302 -        relations = get_unloaded_relations(cnx, cwetype, mapping, fetchtree)
303 -
304 -        if not relations:
305 -            break
306 -
307 -        for path, attr, sub_fetchtree in relations:
308 -            sub_cwetype = attr.datatype
309 -            isarray = wsme.types.isarray(sub_cwetype)
310 -
311 -            if isarray:
312 -                sub_cwetype = sub_cwetype.item_type
313 -
314 -            col_count = iter(xrange(999))
315 -
316 -            etype = sub_cwetype.__etype__
317 -            etypes = None
318 -            if etype == 'Any':
319 -                m = _rtype_re.match(path[-1])
320 -                if m.group('etypes'):
321 -                    etypes = m.group('etypes').split(',')
322 -                    if len(etypes) == 1:
323 -                        etype = etypes[0]
324 -                        etypes = None
325 -
326 -            q = query.Query(cnx.vreg.schema, etype)
327 -
328 -            if etypes:
329 -                q = q.filter(query.Is(*etypes))
330 -
331 -            sub_mapping = get_columns(cnx, sub_cwetype, sub_fetchtree)
332 -            q = add_columns(q, sub_mapping, col_count)
333 -
334 -            # Get all parent objects waiting for relation targets
335 -            objs_by_eid = {}
336 -            for entity in get_entities(entities, path[:-1]):
337 -                objs_by_eid.setdefault(entity.eid, []).append(entity)
338 -
339 -            if not objs_by_eid:
340 -                m = mapping
341 -                for key in path[:-1]:
342 -                    m = m[key]
343 -                m[path[-1]] = sub_mapping
344 -                continue
345 -
346 -            rql, kw = q.torql()
347 -
348 -            rqlst = cnx.vreg.parse(cnx, rql, kw)
349 -
350 -            select = rqlst.children[0]
351 -
352 -            select.set_groupby(list(select.selection))
353 -
354 -            mainvar = select.get_variable('X')
355 -            srcvar = select.get_variable('Y')
356 -
357 -            gr = nodes.Function('GROUP_CONCAT')
358 -            gr.append(nodes.VariableRef(srcvar))
359 -
360 -            select.add_selected(gr)
361 -
362 -            if attr.role == 'subject':
363 -                rel = nodes.make_relation(
364 -                    srcvar, attr.rtype, (mainvar,), nodes.VariableRef)
365 -            else:
366 -                rel = nodes.make_relation(
367 -                    mainvar, attr.rtype, (srcvar,), nodes.VariableRef)
368 -
369 -            select.add_restriction(rel)
370 -
371 -            select.add_restriction(
372 -                nodes.make_constant_restriction(
373 -                    srcvar, 'eid', objs_by_eid.keys(), 'Int'))
374 -
375 -            rset = cnx.execute(select, kw)
376 -
377 -            if isarray:
378 -                for obj in itertools.chain(*objs_by_eid.values()):
379 -                    setattr(obj, attr.key, [])
380 -            for row_i, row in enumerate(rset):
381 -                sub_entity = sub_cwetype.from_row(
382 -                    cnx, rset, row_i, row, sub_mapping, sub_fetchtree)
383 -                eids = [int(eid) for eid in row[-1].split(',')]
384 -
385 -                for obj in itertools.chain(
386 -                        *[objs_by_eid[eid] for eid in eids]):
387 -                    if isarray:
388 -                        getattr(obj, attr.key).append(sub_entity)
389 -                    else:
390 -                        setattr(obj, attr.key, sub_entity)
391 -
392 -            m = mapping
393 -            for key in path[:-1]:
394 -                m = m[key]
395 -            m[path[-1]] = sub_mapping
396 -
397 -    return entities
398 -
399 -
400  class signature(wsme.api.signature):
401      def __call__(self, func):
402          arg_names = self.options.pop('arg_names', None)
403          if arg_names:
404              arg_defaults = self.options.pop('arg_defaults', ())
diff --git a/loader.py b/loader.py
@@ -0,0 +1,380 @@
405 +""" Utilities to load ws types from the database
406 +"""
407 +import itertools
408 +import re
409 +
410 +import wsme.types
411 +
412 +from rql import nodes
413 +
414 +from rqlquery.filter import FilterParser
415 +from rqlquery import query
416 +
417 +from cubes.wsme.types import wsattr
418 +
419 +
420 +def get_fetchable_attributes(user, eschema):
421 +    relations = []
422 +    for rschema in eschema.ordered_relations():
423 +        card = eschema.rdef(rschema.type, takefirst=True).cardinality[0]
424 +        if (card in ('*', '+')
425 +                # cardinality '?' with polymorphic relations raises
426 +                # https://www.cubicweb.org/ticket/4482382
427 +                or card == '?' and len(rschema.objects(eschema)) > 1):
428 +            continue
429 +
430 +        if not rschema.final:
431 +            continue
432 +        if rschema.type in (
433 +                'eid', 'has_text', 'cw_source', 'is'):
434 +            continue
435 +        if rschema.objects()[0].type in ('Password',):
436 +            continue
437 +        rdef = eschema.rdef(rschema)
438 +        if not user.matching_groups(rdef.get_groups('read')):
439 +            continue
440 +        relations.append(rschema)
441 +    return relations
442 +
443 +
444 +def get_fetchable_unary_relations(user, eschema):
445 +    relations = []
446 +    for rschema in eschema.ordered_relations():
447 +        card = eschema.rdef(rschema.type, takefirst=True).cardinality[0]
448 +        if (card in ('*', '+')
449 +                # cardinality '?' with polymorphic relations raises
450 +                # https://www.cubicweb.org/ticket/4482382
451 +                or card == '?' and len(rschema.objects(eschema)) > 1):
452 +            continue
453 +
454 +        if rschema.final:
455 +            continue
456 +
457 +        if rschema.type in (
458 +                'eid', 'has_text', 'cw_source', 'is'):
459 +            continue
460 +        if any(
461 +                not user.matching_groups(es.get_groups('read'))
462 +                for es in rschema.objects(eschema)):
463 +            continue
464 +        relations.append(rschema)
465 +    return relations
466 +
467 +
468 +def get_unfetchable_unary_relations(user, eschema):
469 +    relations = []
470 +
471 +    for rschema in eschema.ordered_relations():
472 +        card = eschema.rdef(rschema.type, takefirst=True).cardinality[0]
473 +        if card == '?' and len(rschema.objects(eschema)) > 1:
474 +            relations.append(rschema)
475 +
476 +    return relations
477 +
478 +
479 +def tree_touch_path(tree, path):
480 +    node = tree
481 +    for name in path:
482 +        node = node.setdefault(name, {})
483 +
484 +
485 +def to_fetchtree(fetchlist, keyonly=False):
486 +    if fetchlist is None:
487 +        if keyonly:
488 +            return {}
489 +        else:
490 +            return {'': {}}
491 +
492 +    fetchtree = {}
493 +
494 +    for rel in fetchlist:
495 +        tree_touch_path(fetchtree, rel.split('.'))
496 +
497 +    return fetchtree
498 +
499 +
500 +def get_columns(cnx, cwetype, fetchtree):
501 +    """ Returns columns to query for filling cwetype
502 +
503 +    Recursively calls itself for inlined relations
504 +
505 +    Returns
506 +    - a mapping for itself and the inlined relations (recursively)
507 +
508 +    """
509 +    mapping = {'eid': None}
510 +
511 +    if fetchtree in (None, {}) or cwetype.__etype__ == 'Any':
512 +        mapping['modification_date'] = None
513 +
514 +    else:
515 +        eschema = cnx.vreg.schema.eschema(cwetype.__etype__)
516 +
517 +        for rtype in get_fetchable_attributes(cnx.user, eschema):
518 +            mapping[rtype.type] = None
519 +
520 +        for rtype in get_fetchable_unary_relations(cnx.user, eschema):
521 +            if not hasattr(cwetype, rtype.type):
522 +                continue
523 +
524 +            tgt_cwetype = getattr(cwetype, rtype.type).datatype
525 +
526 +            mapping[rtype.type] = get_columns(
527 +                cnx, tgt_cwetype, fetchtree.get(rtype.type))
528 +
529 +    return mapping
530 +
531 +
532 +def add_columns(q, mapping, col_counter, prefix=()):
533 +    """ Add columns of the mapping to the query
534 +
535 +    Returns the updated query
536 +    """
537 +    mapping['eid'] = col_counter.next()
538 +
539 +    for key in list(mapping):
540 +        value = mapping[key]
541 +
542 +        if value is None:
543 +            q = q.add_column(prefix + (key,))
544 +            mapping[key] = col_counter.next()
545 +
546 +        elif isinstance(value, dict):
547 +            q = q.add_column(prefix + (key,))
548 +
549 +            q = add_columns(q, value, col_counter, prefix + (key,))
550 +
551 +    return q
552 +
553 +_rtype_re = re.compile(
554 +    r'(?P<rev>\<)?(?P<rtype>[^.[]+)(\[(?P<etypes>[^\]]+)\])?')
555 +
556 +
557 +def get_unloaded_relations(cnx, cwetype, mapping, fetchtree):
558 +    # for each key in fetchtree not present in mapping, check the actual
559 +    # relation type (final or not) and returns it.
560 +
561 +    # for each key in both trees that is a dict with content
562 +    # do a recursive call
563 +
564 +    schema = cnx.vreg.schema
565 +
566 +    relations = []
567 +
568 +    if cwetype.__etype__ == 'Any':
569 +        return ()
570 +
571 +    eschema = schema.eschema(cwetype.__etype__)
572 +    # automatically add the ?* relations because they are expected by the
573 +    # clients but not automatically loaded (see get_fetchable_unary_relations)
574 +    for rschema in get_unfetchable_unary_relations(cnx.user, eschema):
575 +        # make sure the relation exists as an attribute:
576 +        attr = filter(
577 +            lambda a: (
578 +                isinstance(a, wsattr)
579 +                and a.rtype == rschema.type
580 +                and a.role == 'subject'),
581 +            cwetype._wsme_attributes)
582 +        if attr:
583 +            attr = attr[0]
584 +            if attr.key not in fetchtree and attr.key not in mapping:
585 +                relations.append(((attr.key,), attr, None))
586 +
587 +    for key in fetchtree:
588 +        if key in mapping and not(fetchtree[key]):
589 +            continue
590 +
591 +        if key in ('', '*'):
592 +            continue
593 +
594 +        m = _rtype_re.match(key)
595 +        if m is None:
596 +            raise ValueError("Invalid rtype in fetchtree: {}".format(key))
597 +
598 +        name = (m.group('rev') or '') + m.group('rtype')
599 +
600 +        attr = cwetype.attr_by_name(name)
601 +
602 +        if not isinstance(attr, wsattr):
603 +            continue
604 +
605 +        rtype = attr.rtype
606 +
607 +        rschema = schema.rschema(rtype)
608 +
609 +        if rschema.final:
610 +            # TODO handle fetching specific attributes
611 +            # (not in this function obviously)
612 +            continue
613 +
614 +        if key not in mapping:
615 +            relations.append(((key,), attr, fetchtree[key]))
616 +        else:
617 +            relations.extend((
618 +                ((key,) + path, t, f)
619 +                for path, t, f in get_unloaded_relations(
620 +                    cnx,
621 +                    attr.datatype.item_type
622 +                    if wsme.types.isarray(attr.datatype)
623 +                    else attr.datatype,
624 +                    mapping[key],
625 +                    fetchtree[key])
626 +            ))
627 +
628 +    return relations
629 +
630 +
631 +def make_iter(obj_or_iter):
632 +    if obj_or_iter is None:
633 +        return iter(())
634 +    if isinstance(obj_or_iter, (list, dict, set, tuple)):
635 +        return iter(obj_or_iter)
636 +    return iter((obj_or_iter,))
637 +
638 +
639 +def get_entities(roots, path):
640 +    entities = roots
641 +
642 +    for attrname in path:
643 +        if not entities:
644 +            break
645 +        key = entities[0].attr_by_name(attrname).key
646 +        entities = list(itertools.chain(
647 +            *[make_iter(getattr(o, key)) for o in entities]))
648 +
649 +    return entities
650 +
651 +
652 +def load_entities(
653 +        cnx, cwetype,
654 +        orderby=None, query_filter=None, limit=None, offset=None,
655 +        fetchtree=None):
656 +
657 +    q = query.Query(cnx.vreg.schema, cwetype.__etype__)
658 +
659 +    mapping = get_columns(cnx, cwetype, fetchtree)
660 +
661 +    col_count = iter(xrange(999))
662 +
663 +    q = add_columns(q, mapping, col_count)
664 +
665 +    if orderby:
666 +        q = q.orderby(*orderby)
667 +    if query_filter:
668 +        q = q.filter(FilterParser(
669 +            cnx.vreg.schema, cwetype.__etype__, query_filter
670 +        ).parse())
671 +    if limit:
672 +        q = q.limit(limit)
673 +    if offset:
674 +        q = q.offset(offset)
675 +
676 +    rset = q.execute(cnx)
677 +
678 +    entities = [
679 +        cwetype.from_row(cnx, rset, row_i, row, mapping, fetchtree)
680 +        for row_i, row in enumerate(rset)
681 +    ]
682 +
683 +    # nested relation loading
684 +    # scan the mapping and get all unloaded relations that are in fetchtree.
685 +    # Load them, complete the mapping (which is now only a tree of what was
686 +    # loaded), and redo until every relation is loaded
687 +
688 +    while True:
689 +        relations = get_unloaded_relations(cnx, cwetype, mapping, fetchtree)
690 +
691 +        if not relations:
692 +            break
693 +
694 +        for path, attr, sub_fetchtree in relations:
695 +            sub_cwetype = attr.datatype
696 +            isarray = wsme.types.isarray(sub_cwetype)
697 +
698 +            if isarray:
699 +                sub_cwetype = sub_cwetype.item_type
700 +
701 +            col_count = iter(xrange(999))
702 +
703 +            etype = sub_cwetype.__etype__
704 +            etypes = None
705 +            if etype == 'Any':
706 +                m = _rtype_re.match(path[-1])
707 +                if m.group('etypes'):
708 +                    etypes = m.group('etypes').split(',')
709 +                    if len(etypes) == 1:
710 +                        etype = etypes[0]
711 +                        etypes = None
712 +
713 +            q = query.Query(cnx.vreg.schema, etype)
714 +
715 +            if etypes:
716 +                q = q.filter(query.Is(*etypes))
717 +
718 +            sub_mapping = get_columns(cnx, sub_cwetype, sub_fetchtree)
719 +            q = add_columns(q, sub_mapping, col_count)
720 +
721 +            # Get all parent objects waiting for relation targets
722 +            objs_by_eid = {}
723 +            for entity in get_entities(entities, path[:-1]):
724 +                objs_by_eid.setdefault(entity.eid, []).append(entity)
725 +
726 +            if not objs_by_eid:
727 +                m = mapping
728 +                for key in path[:-1]:
729 +                    m = m[key]
730 +                m[path[-1]] = sub_mapping
731 +                continue
732 +
733 +            rql, kw = q.torql()
734 +
735 +            rqlst = cnx.vreg.parse(cnx, rql, kw)
736 +
737 +            select = rqlst.children[0]
738 +
739 +            select.set_groupby(list(select.selection))
740 +
741 +            mainvar = select.get_variable('X')
742 +            srcvar = select.get_variable('Y')
743 +
744 +            gr = nodes.Function('GROUP_CONCAT')
745 +            gr.append(nodes.VariableRef(srcvar))
746 +
747 +            select.add_selected(gr)
748 +
749 +            if attr.role == 'subject':
750 +                rel = nodes.make_relation(
751 +                    srcvar, attr.rtype, (mainvar,), nodes.VariableRef)
752 +            else:
753 +                rel = nodes.make_relation(
754 +                    mainvar, attr.rtype, (srcvar,), nodes.VariableRef)
755 +
756 +            select.add_restriction(rel)
757 +
758 +            select.add_restriction(
759 +                nodes.make_constant_restriction(
760 +                    srcvar, 'eid', objs_by_eid.keys(), 'Int'))
761 +
762 +            rset = cnx.execute(select, kw)
763 +
764 +            if isarray:
765 +                for obj in itertools.chain(*objs_by_eid.values()):
766 +                    setattr(obj, attr.key, [])
767 +            for row_i, row in enumerate(rset):
768 +                sub_entity = sub_cwetype.from_row(
769 +                    cnx, rset, row_i, row, sub_mapping, sub_fetchtree)
770 +                eids = [int(eid) for eid in row[-1].split(',')]
771 +
772 +                for obj in itertools.chain(
773 +                        *[objs_by_eid[eid] for eid in eids]):
774 +                    if isarray:
775 +                        getattr(obj, attr.key).append(sub_entity)
776 +                    else:
777 +                        setattr(obj, attr.key, sub_entity)
778 +
779 +            m = mapping
780 +            for key in path[:-1]:
781 +                m = m[key]
782 +            m[path[-1]] = sub_mapping
783 +
784 +    return entities
diff --git a/test/test_base.py b/test/test_loader.py
@@ -19,20 +19,28 @@
785 
786  import wsme
787 
788  from cubicweb.devtools import testlib
789 
790 -from cubes.wsme.controller import load_entities, get_columns
791 -from cubes.wsme.controller import to_fetchtree
792 +from cubes.wsme.loader import load_entities, get_columns
793 +from cubes.wsme.loader import to_fetchtree
794 +
795 +
796 +class DummyTest(testlib.CubicWebTC):
797 +    def test_dumb(self):
798 +        pass
799 
800 
801 -class WSMETest(testlib.CubicWebTC):
802 +class WSMELoaderTest(testlib.CubicWebTC):
803      @classmethod
804      def init_config(cls, config):
805 -        super(WSMETest, cls).init_config(config)
806 +        super(WSMELoaderTest, cls).init_config(config)
807          config.debugmode = True
808 
809 +    def setUp(self):
810 +        super(WSMELoaderTest, self).setUp()
811 +
812      def test_to_fetchtree(self):
813          self.assertEqual(
814              to_fetchtree([]),
815              {})
816          self.assertEqual(
@@ -77,11 +85,14 @@
817                  'cwuri': None,
818                  'created_by': {'eid': None, 'modification_date': None},
819              })
820 
821      def test_get_columns_ignore_multiple_relations(self):
822 +        print self.vreg.wsme_registry.cwtypes
823 +        print self.vreg.wsme_registry._complex_types
824          CWUser = self.vreg.wsme_registry.lookup('CWUser')
825 +        print CWUser
826 
827          with self.admin_access.repo_cnx() as cnx:
828              m = get_columns(cnx, CWUser, {'in_group': None})
829 
830              self.assertEqual(m, {