wip

authorAdrien Di Mascio <Adrien.DiMascio@logilab.fr>
changesetfb44b455e3d2
branchdefault
phasedraft
hiddenyes
parent revision#200b43c03989 udpate elasticsearch indexes after modifying parent relation (related #13967211)
child revision<not specified>
files modified by this revision
test/__init__.py
test/test_elastic_search.py
test/test_parents.py
test/utils.py
# HG changeset patch
# User Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
# Date 1485973692 -3600
# Wed Feb 01 19:28:12 2017 +0100
# Node ID fb44b455e3d2dc5ee125e1ad1f3ed3ae1d05ef9b
# Parent 200b43c039890762bd12b4b797e1c4875cd5ff24
wip

diff --git a/test/__init__.py b/test/__init__.py
diff --git a/test/test_elastic_search.py b/test/test_elastic_search.py
@@ -3,20 +3,24 @@
1 
2  import sys
3  import unittest
4  from io import StringIO
5  from itertools import repeat
6 +import time
7 
8  from mock import patch
9 
10 +from elasticsearch_dsl import Search
11  from elasticsearch_dsl.faceted_search import FacetedResponse
12 
13  from cubicweb.devtools import testlib
14  from cubicweb.cwconfig import CubicWebConfiguration
15  from cubes.elasticsearch import ccplugin
16  from cubes.elasticsearch.es import (indexable_types,
17                                      fulltext_indexable_rql)
18 +from cubes.elasticsearch.test.utils import RealESTestMixin, BlogFTIAdapter
19 +from cubes.elasticsearch.search_helpers import compose_search
20 
21  # TODO - find a way to configure ElasticSearch as non threaded while running tests
22  # so that the traces show the full stack, not just starting from connection.http_*
23 
24  class ExportElasticSearchTC(testlib.AutoPopulateTest):
@@ -178,7 +182,50 @@
25              self.assertIn('age', rql)
26              self.assertNotIn('eid', rql)
27              self.assertEqual(rql.count('modification_date'), 1)
28 
29 
30 +class ReindexOnRelationTests(RealESTestMixin, testlib.CubicWebTC):
31 +
32 +    def test_es_hooks_modify_relation(self):
33 +        with self.admin_access.cnx() as cnx:
34 +            with self.temporary_appobjects(BlogFTIAdapter):
35 +                indexer = cnx.vreg['es'].select('indexer', cnx)
36 +                indexer.create_index(custom_settings={
37 +                    'mappings': {
38 +                        'BlogEntry': {'_parent': {"type": "Blog"}},
39 +                    }
40 +                })
41 +                blog1 = cnx.create_entity('Blog', title=u'Blog')
42 +                entity = cnx.create_entity('BlogEntry',
43 +                                           title=u'Article about stuff',
44 +                                           content=u'yippee',
45 +                                           entry_of=blog1)
46 +                blog2 = cnx.create_entity('Blog', title=u'Blog')
47 +                cnx.commit()
48 +                time.sleep(2)  # TODO find a way to have synchronous operations in unittests
49 +                search = compose_search(Search(index=self.config['index-name'],
50 +                                               doc_type='Blog'),
51 +                                        'yippee',
52 +                                        parents_for="BlogEntry",
53 +                                        fields=['_all'])
54 +                results = search.execute()
55 +                self.assertEquals(len(results), 1)
56 +                self.assertCountEqual([hit.eid for hit in results],
57 +                                      [blog1.eid])
58 +                blog2.cw_set(reverse_entry_of=entity)
59 +                cnx.commit()
60 +                time.sleep(2)  # TODO find a way to have synchronous operations in unittests
61 +                search = compose_search(Search(index=self.config['index-name'],
62 +                                               doc_type='Blog'),
63 +                                        'yippee',
64 +                                        parents_for="BlogEntry",
65 +                                        fields=['_all'])
66 +                results = search.execute()
67 +                self.assertEquals(len(results), 2)
68 +                self.assertCountEqual([hit.eid for hit in results],
69 +                                      [blog1.eid, blog2.eid])
70 +
71 +
72 +
73  if __name__ == '__main__':
74      unittest.main()
diff --git a/test/test_parents.py b/test/test_parents.py
@@ -1,65 +1,25 @@
75  from __future__ import print_function
76 
77  import time
78  import unittest
79 -import httplib
80 
81  from elasticsearch_dsl import Search
82 -from elasticsearch_dsl.connections import connections
83 
84  from cubicweb.devtools import testlib
85 -from cubicweb.cwconfig import CubicWebConfiguration
86 -from cubicweb.predicates import is_instance
87 
88  from cubes.elasticsearch.search_helpers import compose_search
89 
90 -from cubes.elasticsearch.es import CUSTOM_ATTRIBUTES
91 -from cubes.elasticsearch.entities import IFullTextIndexSerializable
92 -
93 -CUSTOM_ATTRIBUTES['Blog'] = ('title',)
94 -
95 -
96 -class BlogFTIAdapter(IFullTextIndexSerializable):
97 -    __select__ = (IFullTextIndexSerializable.__select__ &
98 -                  is_instance('BlogEntry'))
99 -
100 -    def update_parent_info(self, data, entity):
101 -        data['parent'] = entity.entry_of[0].eid
102 +from cubes.elasticsearch.test.utils import RealESTestMixin, BlogFTIAdapter
103 
104 
105 -class ParentsSearchTC(testlib.CubicWebTC):
106 -
107 -    @classmethod
108 -    def setUpClass(cls):
109 -        try:
110 -            httplib.HTTPConnection('localhost:9200').request('GET', '/')
111 -        except:
112 -            raise unittest.SkipTest('No ElasticSearch on localhost, skipping test')
113 -        super(ParentsSearchTC, cls).setUpClass()
114 +class ParentsSearchTC(RealESTestMixin, testlib.CubicWebTC):
115 
116 -    def setup_database(self):
117 -        super(ParentsSearchTC, self).setup_database()
118 -        self.orig_config_for = CubicWebConfiguration.config_for
119 -        config_for = lambda appid: self.config  # noqa
120 -        CubicWebConfiguration.config_for = staticmethod(config_for)
121 -        self.config['elasticsearch-locations'] = 'http://localhost:9200'
122 -        # TODO unique ID to avoid collision
123 -        self.config['index-name'] = 'unittest_index_name'
124 -        # remove default connection if there's one
125 -        try:
126 -            connections.remove_connection('default')
127 -        except KeyError:
128 -            pass
129 -
130 -    @unittest.skip('TODO')
131      def test_parent_search(self):
132 -        # self.vid_validators['esearch'] = lambda: None
133          with self.admin_access.cnx() as cnx:
134              with self.temporary_appobjects(BlogFTIAdapter):
135                  indexer = cnx.vreg['es'].select('indexer', cnx)
136 -                indexer.get_connection()
137                  indexer.create_index(custom_settings={
138                      'mappings': {
139                          'BlogEntry': {'_parent': {"type": "Blog"}},
140                      }
141                  })
@@ -88,42 +48,8 @@
142                                          fuzzy=True)
143                  self.assertEquals(len(search.execute()), number_of_results)
144                  self.assertEquals(search.execute().to_dict()['hits']['hits'][0]['_source']['title'],
145                                    first_result)
146 
147 -    def test_es_hooks_modify_relation(self):
148 -        with self.admin_access.cnx() as cnx:
149 -            with self.temporary_appobjects(BlogFTIAdapter):
150 -                indexer = cnx.vreg['es'].select('indexer', cnx)
151 -                indexer.get_connection()
152 -                indexer.create_index(custom_settings={
153 -                    'mappings': {
154 -                        'BlogEntry': {'_parent': {"type": "Blog"}},
155 -                    }
156 -                })
157 -                blog1 = cnx.create_entity('Blog', title=u'Blog')
158 -                entity = cnx.create_entity('BlogEntry',
159 -                                           title=u'Article about stuff',
160 -                                           content=u'content herer',
161 -                                           entry_of=blog1)
162 -                blog2 = cnx.create_entity('Blog', title=u'Blog')
163 -                cnx.commit()
164 -                time.sleep(2)  # TODO find a way to have synchronous operations in unittests
165 -
166 -                blog2.cw_set(reverse_entry_of=entity)
167 -                cnx.commit()
168 -                search = Search(index=self.config['index-name'],
169 -                                doc_type='Blog').query(
170 -                                    "match", eid=entity.eid)
171 -                print(search.execute().to_dict()['hits']['hits'])
172 -
173 -    def tearDown(self):
174 -        with self.admin_access.cnx() as cnx:
175 -            indexer = cnx.vreg['es'].select('indexer', cnx)
176 -            es = indexer.get_connection()
177 -            es.indices.delete(self.config['index-name'])
178 -        super(ParentsSearchTC, self).tearDown()
179 -
180 
181  if __name__ == '__main__':
182 -    from logilab.common.testlib import unittest_main
183 -    unittest_main()
184 +    unittest.main()
diff --git a/test/utils.py b/test/utils.py
@@ -0,0 +1,52 @@
185 +import unittest
186 +import httplib
187 +
188 +from elasticsearch_dsl.connections import connections
189 +
190 +from cubicweb.predicates import is_instance
191 +
192 +from cubes.elasticsearch.es import CUSTOM_ATTRIBUTES
193 +from cubes.elasticsearch.entities import IFullTextIndexSerializable
194 +
195 +
196 +CUSTOM_ATTRIBUTES['Blog'] = ('title',)
197 +
198 +
199 +class BlogFTIAdapter(IFullTextIndexSerializable):
200 +    __select__ = (IFullTextIndexSerializable.__select__ &
201 +                  is_instance('BlogEntry'))
202 +
203 +    def update_parent_info(self, data, entity):
204 +        data['parent'] = entity.entry_of[0].eid
205 +
206 +
207 +class RealESTestMixin(object):
208 +
209 +    @classmethod
210 +    def setUpClass(cls):
211 +        try:
212 +            httplib.HTTPConnection('localhost:9200').request('GET', '/')
213 +        except:
214 +            raise unittest.SkipTest('No ElasticSearch on localhost, skipping test')
215 +        super(RealESTestMixin, cls).setUpClass()
216 +
217 +    def setup_database(self):
218 +        super(RealESTestMixin, self).setup_database()
219 +        self.config.global_set_option('elasticsearch-locations',
220 +                                      'http://localhost:9200')
221 +        self.config.global_set_option('index-name',
222 +                                      'unittest_index_name')
223 +
224 +    def tearDown(self):
225 +        try:
226 +            with self.admin_access.cnx() as cnx:
227 +                indexer = cnx.vreg['es'].select('indexer', cnx)
228 +                es = indexer.get_connection()
229 +                es.indices.delete(self.config['index-name'])
230 +        finally:
231 +            # remove default connection if there's one
232 +            try:
233 +                connections.remove_connection('default')
234 +            except KeyError:
235 +                pass
236 +            super(RealESTestMixin, self).tearDown()