udpate elasticsearch indexes after modifying parent relation

1      return entity.cw_etype in indexable_types(entity._cw.vreg.schema) or \
2          entity.cw_etype in CUSTOM_ATTRIBUTES
3 
4 
5  class ContentUpdateIndexES(hook.Hook):
6 -
7      """detect content change and updates ES indexing"""
8 
9      __regid__ = 'elasticsearch.contentupdatetoes'
10      __select__ = hook.Hook.__select__ & score_entity(entity_indexable)
11      events = ('after_update_entity', 'after_add_entity')

12          if self.entity.cw_etype == 'File':
13              return  # FIXME hack!
14          IndexEsOperation.get_instance(self._cw).add_data(self.entity)
15 
16 
17 +class RelationsUpdateIndexES(hook.Hook):
18 +    """detect relations changes and updates ES indexing"""
19 +
20 +    __regid__ = 'elasticsearch.relationsupdatetoes'
21 +    events = ('after_add_relation', 'before_delete_relation')
22 +    category = 'es'
23 +
24 +    def __call__(self):
25 +        # XXX add a selector for object and subject
26 +        for entity in (self._cw.entity_from_eid(self.eidfrom),
27 +                       self._cw.entity_from_eid(self.eidto)):
28 +            cw_etype = entity.cw_etype
29 +            if cw_etype == 'File':
30 +                return  # FIXME hack!
31 +            if (cw_etype in indexable_types(entity._cw.vreg.schema) or
32 +                    cw_etype in CUSTOM_ATTRIBUTES):
33 +                IndexEsOperation.get_instance(self._cw).add_data(entity)
34 +
35 +
36  class IndexEsOperation(hook.DataOperationMixIn, hook.Operation):
37 
38      def precommit_event(self):
39          indexer = self.cnx.vreg['es'].select('indexer', self.cnx)
40          es = indexer.get_connection()
41          if es is None or not self.cnx.vreg.config['index-name']:
42 -            log.info('no connection to ES (not configured) skip ES indexing')
43 +            log.error('no connection to ES (not configured) skip ES indexing')
44              return
45          for entity in self.get_data():
46              rql = fulltext_indexable_rql(entity.cw_etype,
47                                           entity._cw.vreg.schema,
48                                           eid=entity.eid)

49 +import unittest
50 +import time
51 +
52 +from elasticsearch_dsl import Search
53 +
54 +from cubicweb.devtools import testlib
55 +
56 +from cubes.elasticsearch.testutils import RealESTestMixin, BlogFTIAdapter
57 +from cubes.elasticsearch.search_helpers import compose_search
58 +
59 +
60 +class ReindexOnRelationTests(RealESTestMixin, testlib.CubicWebTC):
61 +
62 +    def test_es_hooks_modify_relation(self):
63 +        with self.admin_access.cnx() as cnx:
64 +            with self.temporary_appobjects(BlogFTIAdapter):
65 +                indexer = cnx.vreg['es'].select('indexer', cnx)
66 +                indexer.create_index(custom_settings={
67 +                    'mappings': {
68 +                        'BlogEntry': {'_parent': {"type": "Blog"}},
69 +                    }
70 +                })
71 +                blog1 = cnx.create_entity('Blog', title=u'Blog')
72 +                entity = cnx.create_entity('BlogEntry',
73 +                                           title=u'Article about stuff',
74 +                                           content=u'yippee',
75 +                                           entry_of=blog1)
76 +                blog2 = cnx.create_entity('Blog', title=u'Blog')
77 +                cnx.commit()
78 +                time.sleep(2)  # TODO find a way to have synchronous operations in unittests
79 +                search = compose_search(Search(index=self.config['index-name'],
80 +                                               doc_type='Blog'),
81 +                                        'yippee',
82 +                                        parents_for="BlogEntry",
83 +                                        fields=['_all'])
84 +                results = search.execute()
85 +                self.assertEquals(len(results), 1)
86 +                self.assertCountEqual([hit.eid for hit in results],
87 +                                      [blog1.eid])
88 +                blog2.cw_set(reverse_entry_of=entity)
89 +                cnx.commit()
90 +                time.sleep(2)  # TODO find a way to have synchronous operations in unittests
91 +                search = compose_search(Search(index=self.config['index-name'],
92 +                                               doc_type='Blog'),
93 +                                        'yippee',
94 +                                        parents_for="BlogEntry",
95 +                                        fields=['_all'])
96 +                results = search.execute()
97 +                self.assertEquals(len(results), 2)
98 +                self.assertCountEqual([hit.eid for hit in results],
99 +                                      [blog1.eid, blog2.eid])
100 +
101 +
102 +if __name__ == '__main__':
103 +    unittest.main()

104  from __future__ import print_function
105 
106  import time
107  import unittest
108 -import httplib
109 
110  from elasticsearch_dsl import Search
111 -from elasticsearch_dsl.connections import connections
112 
113  from cubicweb.devtools import testlib
114 -from cubicweb.cwconfig import CubicWebConfiguration
115 -from cubicweb.predicates import is_instance
116 
117  from cubes.elasticsearch.search_helpers import compose_search
118 
119 -from cubes.elasticsearch.es import CUSTOM_ATTRIBUTES
120 -from cubes.elasticsearch.entities import IFullTextIndexSerializable
121 -
122 -CUSTOM_ATTRIBUTES['Blog'] = ('title',)
123 -
124 -
125 -class BlogFTIAdapter(IFullTextIndexSerializable):
126 -    __select__ = (IFullTextIndexSerializable.__select__ &
127 -                  is_instance('BlogEntry'))
128 -
129 -    def update_parent_info(self, data, entity):
130 -        data['parent'] = entity.entry_of[0].eid
131 +from cubes.elasticsearch.testutils import RealESTestMixin, BlogFTIAdapter
132 
133 
134 -class ParentsSearchTC(testlib.CubicWebTC):
135 -
136 -    @classmethod
137 -    def setUpClass(cls):
138 -        try:
139 -            httplib.HTTPConnection('localhost:9200').request('GET', '/')
140 -        except:
141 -            raise unittest.SkipTest('No ElasticSearch on localhost, skipping test')
142 -        super(ParentsSearchTC, cls).setUpClass()
143 -
144 -    def setup_database(self):
145 -        super(ParentsSearchTC, self).setup_database()
146 -        self.orig_config_for = CubicWebConfiguration.config_for
147 -        config_for = lambda appid: self.config  # noqa
148 -        CubicWebConfiguration.config_for = staticmethod(config_for)
149 -        self.config['elasticsearch-locations'] = 'http://localhost:9200'
150 -        # TODO unique ID to avoid collision
151 -        self.config['index-name'] = 'unittest_index_name'
152 -        # remove default connection if there's one
153 -        try:
154 -            connections.remove_connection('default')
155 -        except KeyError:
156 -            pass
157 +class ParentsSearchTC(RealESTestMixin, testlib.CubicWebTC):
158 
159      def test_parent_search(self):
160 -        # self.vid_validators['esearch'] = lambda: None
161          with self.admin_access.cnx() as cnx:
162              with self.temporary_appobjects(BlogFTIAdapter):
163                  indexer = cnx.vreg['es'].select('indexer', cnx)
164 -                indexer.get_connection()
165                  indexer.create_index(custom_settings={
166                      'mappings': {
167                          'BlogEntry': {'_parent': {"type": "Blog"}},
168                      }
169                  })

170                                          fuzzy=True)
171                  self.assertEquals(len(search.execute()), number_of_results)
172                  self.assertEquals(search.execute().to_dict()['hits']['hits'][0]['_source']['title'],
173                                    first_result)
174 
175 -    def tearDown(self):
176 -        with self.admin_access.cnx() as cnx:
177 -            indexer = cnx.vreg['es'].select('indexer', cnx)
178 -            es = indexer.get_connection()
179 -            es.indices.delete(self.config['index-name'])
180 -        super(ParentsSearchTC, self).tearDown()
181 -
182 
183  if __name__ == '__main__':
184 -    from logilab.common.testlib import unittest_main
185 -    unittest_main()
186 +    unittest.main()

187 +import unittest
188 +import httplib
189 +
190 +from elasticsearch_dsl.connections import connections
191 +
192 +from cubicweb.predicates import is_instance
193 +
194 +from cubes.elasticsearch.es import CUSTOM_ATTRIBUTES
195 +from cubes.elasticsearch.entities import IFullTextIndexSerializable
196 +
197 +
198 +CUSTOM_ATTRIBUTES['Blog'] = ('title',)
199 +
200 +
201 +class BlogFTIAdapter(IFullTextIndexSerializable):
202 +    __select__ = (IFullTextIndexSerializable.__select__ &
203 +                  is_instance('BlogEntry'))
204 +
205 +    def update_parent_info(self, data, entity):
206 +        data['parent'] = entity.entry_of[0].eid
207 +
208 +
209 +class RealESTestMixin(object):
210 +
211 +    @classmethod
212 +    def setUpClass(cls):
213 +        try:
214 +            httplib.HTTPConnection('localhost:9200').request('GET', '/')
215 +        except:
216 +            raise unittest.SkipTest('No ElasticSearch on localhost, skipping test')
217 +        super(RealESTestMixin, cls).setUpClass()
218 +
219 +    def setup_database(self):
220 +        super(RealESTestMixin, self).setup_database()
221 +        self.config.global_set_option('elasticsearch-locations',
222 +                                      'http://localhost:9200')
223 +        self.config.global_set_option('index-name',
224 +                                      'unittest_index_name')
225 +
226 +    def tearDown(self):
227 +        try:
228 +            with self.admin_access.cnx() as cnx:
229 +                indexer = cnx.vreg['es'].select('indexer', cnx)
230 +                es = indexer.get_connection()
231 +                es.indices.delete(self.config['index-name'])
232 +        finally:
233 +            # remove default connection if there's one
234 +            try:
235 +                connections.remove_connection('default')
236 +            except KeyError:
237 +                pass
238 +            super(RealESTestMixin, self).tearDown()