# HG changeset patch
# User Katia Saurfelt <katia.saurfelt@logilab.fr>
# Date 1485973692 -3600
# Wed Feb 01 19:28:12 2017 +0100
# Node ID a571130d47bff96357979acb496ff18852e34e12
# Parent ac5f8d85cf00f9cbc9a6b544b75d4161182c1a9d
udpate elasticsearch indexes after modifying parent relation
closes #17052625
# User Katia Saurfelt <katia.saurfelt@logilab.fr>
# Date 1485973692 -3600
# Wed Feb 01 19:28:12 2017 +0100
# Node ID a571130d47bff96357979acb496ff18852e34e12
# Parent ac5f8d85cf00f9cbc9a6b544b75d4161182c1a9d
udpate elasticsearch indexes after modifying parent relation
closes #17052625
@@ -33,11 +33,10 @@
1 return entity.cw_etype in indexable_types(entity._cw.vreg.schema) or \ 2 entity.cw_etype in CUSTOM_ATTRIBUTES 3 4 5 class ContentUpdateIndexES(hook.Hook): 6 - 7 """detect content change and updates ES indexing""" 8 9 __regid__ = 'elasticsearch.contentupdatetoes' 10 __select__ = hook.Hook.__select__ & score_entity(entity_indexable) 11 events = ('after_update_entity', 'after_add_entity')
@@ -47,17 +46,36 @@
12 if self.entity.cw_etype == 'File': 13 return # FIXME hack! 14 IndexEsOperation.get_instance(self._cw).add_data(self.entity) 15 16 17 +class RelationsUpdateIndexES(hook.Hook): 18 + """detect relations changes and updates ES indexing""" 19 + 20 + __regid__ = 'elasticsearch.relationsupdatetoes' 21 + events = ('after_add_relation', 'before_delete_relation') 22 + category = 'es' 23 + 24 + def __call__(self): 25 + # XXX add a selector for object and subject 26 + for entity in (self._cw.entity_from_eid(self.eidfrom), 27 + self._cw.entity_from_eid(self.eidto)): 28 + cw_etype = entity.cw_etype 29 + if cw_etype == 'File': 30 + return # FIXME hack! 31 + if (cw_etype in indexable_types(entity._cw.vreg.schema) or 32 + cw_etype in CUSTOM_ATTRIBUTES): 33 + IndexEsOperation.get_instance(self._cw).add_data(entity) 34 + 35 + 36 class IndexEsOperation(hook.DataOperationMixIn, hook.Operation): 37 38 def precommit_event(self): 39 indexer = self.cnx.vreg['es'].select('indexer', self.cnx) 40 es = indexer.get_connection() 41 if es is None or not self.cnx.vreg.config['index-name']: 42 - log.info('no connection to ES (not configured) skip ES indexing') 43 + log.error('no connection to ES (not configured) skip ES indexing') 44 return 45 for entity in self.get_data(): 46 rql = fulltext_indexable_rql(entity.cw_etype, 47 entity._cw.vreg.schema, 48 eid=entity.eid)
@@ -0,0 +1,55 @@
49 +import unittest 50 +import time 51 + 52 +from elasticsearch_dsl import Search 53 + 54 +from cubicweb.devtools import testlib 55 + 56 +from cubes.elasticsearch.testutils import RealESTestMixin, BlogFTIAdapter 57 +from cubes.elasticsearch.search_helpers import compose_search 58 + 59 + 60 +class ReindexOnRelationTests(RealESTestMixin, testlib.CubicWebTC): 61 + 62 + def test_es_hooks_modify_relation(self): 63 + with self.admin_access.cnx() as cnx: 64 + with self.temporary_appobjects(BlogFTIAdapter): 65 + indexer = cnx.vreg['es'].select('indexer', cnx) 66 + indexer.create_index(custom_settings={ 67 + 'mappings': { 68 + 'BlogEntry': {'_parent': {"type": "Blog"}}, 69 + } 70 + }) 71 + blog1 = cnx.create_entity('Blog', title=u'Blog') 72 + entity = cnx.create_entity('BlogEntry', 73 + title=u'Article about stuff', 74 + content=u'yippee', 75 + entry_of=blog1) 76 + blog2 = cnx.create_entity('Blog', title=u'Blog') 77 + cnx.commit() 78 + time.sleep(2) # TODO find a way to have synchronous operations in unittests 79 + search = compose_search(Search(index=self.config['index-name'], 80 + doc_type='Blog'), 81 + 'yippee', 82 + parents_for="BlogEntry", 83 + fields=['_all']) 84 + results = search.execute() 85 + self.assertEquals(len(results), 1) 86 + self.assertCountEqual([hit.eid for hit in results], 87 + [blog1.eid]) 88 + blog2.cw_set(reverse_entry_of=entity) 89 + cnx.commit() 90 + time.sleep(2) # TODO find a way to have synchronous operations in unittests 91 + search = compose_search(Search(index=self.config['index-name'], 92 + doc_type='Blog'), 93 + 'yippee', 94 + parents_for="BlogEntry", 95 + fields=['_all']) 96 + results = search.execute() 97 + self.assertEquals(len(results), 2) 98 + self.assertCountEqual([hit.eid for hit in results], 99 + [blog1.eid, blog2.eid]) 100 + 101 + 102 +if __name__ == '__main__': 103 + unittest.main()
@@ -1,64 +1,25 @@
104 from __future__ import print_function 105 106 import time 107 import unittest 108 -import httplib 109 110 from elasticsearch_dsl import Search 111 -from elasticsearch_dsl.connections import connections 112 113 from cubicweb.devtools import testlib 114 -from cubicweb.cwconfig import CubicWebConfiguration 115 -from cubicweb.predicates import is_instance 116 117 from cubes.elasticsearch.search_helpers import compose_search 118 119 -from cubes.elasticsearch.es import CUSTOM_ATTRIBUTES 120 -from cubes.elasticsearch.entities import IFullTextIndexSerializable 121 - 122 -CUSTOM_ATTRIBUTES['Blog'] = ('title',) 123 - 124 - 125 -class BlogFTIAdapter(IFullTextIndexSerializable): 126 - __select__ = (IFullTextIndexSerializable.__select__ & 127 - is_instance('BlogEntry')) 128 - 129 - def update_parent_info(self, data, entity): 130 - data['parent'] = entity.entry_of[0].eid 131 +from cubes.elasticsearch.testutils import RealESTestMixin, BlogFTIAdapter 132 133 134 -class ParentsSearchTC(testlib.CubicWebTC): 135 - 136 - @classmethod 137 - def setUpClass(cls): 138 - try: 139 - httplib.HTTPConnection('localhost:9200').request('GET', '/') 140 - except: 141 - raise unittest.SkipTest('No ElasticSearch on localhost, skipping test') 142 - super(ParentsSearchTC, cls).setUpClass() 143 - 144 - def setup_database(self): 145 - super(ParentsSearchTC, self).setup_database() 146 - self.orig_config_for = CubicWebConfiguration.config_for 147 - config_for = lambda appid: self.config # noqa 148 - CubicWebConfiguration.config_for = staticmethod(config_for) 149 - self.config['elasticsearch-locations'] = 'http://localhost:9200' 150 - # TODO unique ID to avoid collision 151 - self.config['index-name'] = 'unittest_index_name' 152 - # remove default connection if there's one 153 - try: 154 - connections.remove_connection('default') 155 - except KeyError: 156 - pass 157 +class ParentsSearchTC(RealESTestMixin, testlib.CubicWebTC): 158 159 def test_parent_search(self): 160 - # self.vid_validators['esearch'] = lambda: None 161 with self.admin_access.cnx() as cnx: 162 with self.temporary_appobjects(BlogFTIAdapter): 163 indexer = cnx.vreg['es'].select('indexer', cnx) 164 - indexer.get_connection() 165 indexer.create_index(custom_settings={ 166 'mappings': { 167 'BlogEntry': {'_parent': {"type": "Blog"}}, 168 } 169 })
@@ -87,16 +48,8 @@
170 fuzzy=True) 171 self.assertEquals(len(search.execute()), number_of_results) 172 self.assertEquals(search.execute().to_dict()['hits']['hits'][0]['_source']['title'], 173 first_result) 174 175 - def tearDown(self): 176 - with self.admin_access.cnx() as cnx: 177 - indexer = cnx.vreg['es'].select('indexer', cnx) 178 - es = indexer.get_connection() 179 - es.indices.delete(self.config['index-name']) 180 - super(ParentsSearchTC, self).tearDown() 181 - 182 183 if __name__ == '__main__': 184 - from logilab.common.testlib import unittest_main 185 - unittest_main() 186 + unittest.main()
@@ -0,0 +1,52 @@
187 +import unittest 188 +import httplib 189 + 190 +from elasticsearch_dsl.connections import connections 191 + 192 +from cubicweb.predicates import is_instance 193 + 194 +from cubes.elasticsearch.es import CUSTOM_ATTRIBUTES 195 +from cubes.elasticsearch.entities import IFullTextIndexSerializable 196 + 197 + 198 +CUSTOM_ATTRIBUTES['Blog'] = ('title',) 199 + 200 + 201 +class BlogFTIAdapter(IFullTextIndexSerializable): 202 + __select__ = (IFullTextIndexSerializable.__select__ & 203 + is_instance('BlogEntry')) 204 + 205 + def update_parent_info(self, data, entity): 206 + data['parent'] = entity.entry_of[0].eid 207 + 208 + 209 +class RealESTestMixin(object): 210 + 211 + @classmethod 212 + def setUpClass(cls): 213 + try: 214 + httplib.HTTPConnection('localhost:9200').request('GET', '/') 215 + except: 216 + raise unittest.SkipTest('No ElasticSearch on localhost, skipping test') 217 + super(RealESTestMixin, cls).setUpClass() 218 + 219 + def setup_database(self): 220 + super(RealESTestMixin, self).setup_database() 221 + self.config.global_set_option('elasticsearch-locations', 222 + 'http://localhost:9200') 223 + self.config.global_set_option('index-name', 224 + 'unittest_index_name') 225 + 226 + def tearDown(self): 227 + try: 228 + with self.admin_access.cnx() as cnx: 229 + indexer = cnx.vreg['es'].select('indexer', cnx) 230 + es = indexer.get_connection() 231 + es.indices.delete(self.config['index-name']) 232 + finally: 233 + # remove default connection if there's one 234 + try: 235 + connections.remove_connection('default') 236 + except KeyError: 237 + pass 238 + super(RealESTestMixin, self).tearDown()