allow overriding of ES id attriubte

Default is to use 'eid' but one may want to use something else (stable id, ark, etc.)

closes #17079908

authorAdrien Di Mascio <Adrien.DiMascio@logilab.fr>
changesetdfaf162a655e
branchdefault
phasedraft
hiddenyes
parent revision#a9a0b699f3da attr indexation should not depend on its presence in cw_attr_cache
child revision#c1a4bddf010b use property to override elasticsearch id, #b86ac43aea7d [refactoring] split IFullTextIndexSerializable.serailize method to ease overriding, #eade447e6a3c [refactoring] split IFullTextIndexSerializable.serailize method to ease overriding
files modified by this revision
cubicweb_elasticsearch/ccplugin.py
cubicweb_elasticsearch/entities.py
cubicweb_elasticsearch/hooks.py
cubicweb_elasticsearch/testutils.py
test/test_ifulltextadapter.py
# HG changeset patch
# User Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
# Date 1495456104 -7200
# Mon May 22 14:28:24 2017 +0200
# Node ID dfaf162a655edcf7849dbcac9bd1f1b1b31516c2
# Parent a9a0b699f3da42a7b5686a24232d63caded8b3dd
allow overriding of ES id attriubte

Default is to use 'eid' but one may want to use something
else (stable id, ark, etc.)

closes #17079908

diff --git a/cubicweb_elasticsearch/ccplugin.py b/cubicweb_elasticsearch/ccplugin.py
@@ -115,11 +115,11 @@
1                      # IFullTextIndex serializer , therefor the "id" and
2                      # "doc_type" in kwargs bellow must be container data.
3                      data = {'_op_type': 'index',
4                              '_index': index_name or cnx.vreg.config['index-name'],
5                              '_type': json['cw_etype'],
6 -                            '_id': json['eid'],
7 +                            '_id': json[serializer.es_id_attr],
8                              '_source': json
9                              }
10                      self.customize_data(data)
11                      yield data
12 
diff --git a/cubicweb_elasticsearch/entities.py b/cubicweb_elasticsearch/entities.py
@@ -88,10 +88,11 @@
13      directly serialized to e.g. JSON.
14      """
15 
16      __regid__ = 'IFullTextIndexSerializable'
17      __select__ = is_instance('Any')
18 +    es_id_attr = 'eid'
19      custom_indexable_attributes = ()
20      skip_indexable_attributes = ()
21 
22      @cachedproperty
23      def fulltext_indexable_attributes(self):
diff --git a/cubicweb_elasticsearch/hooks.py b/cubicweb_elasticsearch/hooks.py
@@ -96,11 +96,11 @@
24                  continue
25              kwargs['body'] = json
26              # Entities with fulltext_containers relations return their container
27              # IFullTextIndex serializer, therefore the "id" and "doc_type" in
28              # kwargs below must be container data.
29 -            kwargs['id'] = json['eid']
30 +            kwargs['id'] = json[serializer.es_id_attr]
31              kwargs['doc_type'] = getattr(serializer, 'doc_type', json['cw_etype'])
32              try:
33                  # TODO option for async ?
34                  es.index(**kwargs)
35              except (ConnectionError, ProtocolError) as exc:
diff --git a/cubicweb_elasticsearch/testutils.py b/cubicweb_elasticsearch/testutils.py
@@ -18,10 +18,16 @@
36      __select__ = (IFullTextIndexSerializable.__select__ &
37                    is_instance('Blog'))
38      custom_indexable_attributes = ('title', )
39 
40 
41 +class PersonFTIAdapter(IFullTextIndexSerializable):
42 +    __select__ = (IFullTextIndexSerializable.__select__ &
43 +                  is_instance('Person'))
44 +    es_id_attr = 'age'
45 +
46 +
47  class RealESTestMixin(object):
48 
49      @classmethod
50      def setUpClass(cls):
51          try:
diff --git a/test/test_ifulltextadapter.py b/test/test_ifulltextadapter.py
@@ -3,11 +3,13 @@
52  from mock import patch
53 
54  from cubicweb.devtools import testlib
55  from cubicweb.cwconfig import CubicWebConfiguration
56 
57 -from cubes.elasticsearch.testutils import BlogFTIAdapter, BlogEntryFTIAdapter
58 +from cubicweb_elasticsearch.testutils import (BlogFTIAdapter,
59 +                                              BlogEntryFTIAdapter,
60 +                                              PersonFTIAdapter)
61 
62 
63  class IFullTextIndexSerializableTC(testlib.CubicWebTC):
64 
65      def setup_database(self):
@@ -65,8 +67,41 @@
66                          ('cwuri', bentry.cwuri),
67                          ('title', u'Programme')):
68                      self.assertEqual(kwargs['body'][arg_name], expected_value)
69                  self.assertFalse('content_format' in kwargs['body'])
70 
71 +    @patch('elasticsearch.client.indices.IndicesClient.create')
72 +    @patch('elasticsearch.client.indices.IndicesClient.exists')
73 +    @patch('elasticsearch.client.Elasticsearch.index')
74 +    def test_custom_es_id_attr(self, create, exists, index):
75 +        """check custom es_eid_attr is used in es document"""
76 +        with self.admin_access.repo_cnx() as cnx:
77 +            with self.temporary_appobjects(PersonFTIAdapter):
78 +                cnx.create_entity('Person', age=123456, name=u'Jean')
79 +                cnx.commit()
80 +                indexer = cnx.vreg['es'].select('indexer', cnx)
81 +                es = indexer.get_connection()
82 +                self.assertTrue(es.index.called)
83 +                args, kwargs = es.index.call_args
84 +                # make sure age was used as custom es document id
85 +                self.assertEqual(kwargs['id'], 123456)
86 +
87 +
88 +class SerializationTests(testlib.CubicWebTC):
89 +
90 +    def test_person_serialization(self):
91 +        with self.admin_access.cnx() as cnx:
92 +            jean = cnx.create_entity('Person', age=12, name=u'Jean')
93 +            serializer = jean.cw_adapt_to('IFullTextIndexSerializable')
94 +            self.assertEqual(serializer.serialize(), {
95 +                'name': u'Jean',
96 +                'age': 12,
97 +                'cw_etype': u'Person',
98 +                'eid': jean.eid,
99 +                'cwuri': jean.cwuri,
100 +                'creation_date': jean.creation_date,
101 +                'modification_date': jean.modification_date,
102 +            })
103 +
104 
105  if __name__ == '__main__':
106      unittest.main()