[pkg] make elasticsearch a new-layout-style cube

(closes #17071989)

authorAdrien Di Mascio <Adrien.DiMascio@logilab.fr>
changesete6392f5058c4
branchdefault
phasepublic
hiddenno
parent revision#14a2e29584cd [views] safety belt around page argument handling (if not an int)
child revision#af6334be7b14 [IFullTextIndexSerializable] use fulltext_indexable_attributes attribute to update cw_attr_cache, #074421eb0f64 [test] unittest on hook
files modified by this revision
MANIFEST.in
__init__.py
__pkginfo__.py
ccplugin.py
cubicweb-elasticsearch.spec
cubicweb_elasticsearch/__init__.py
cubicweb_elasticsearch/__pkginfo__.py
cubicweb_elasticsearch/ccplugin.py
cubicweb_elasticsearch/entities.py
cubicweb_elasticsearch/es.py
cubicweb_elasticsearch/hooks.py
cubicweb_elasticsearch/i18n/en.po
cubicweb_elasticsearch/i18n/es.po
cubicweb_elasticsearch/i18n/fr.po
cubicweb_elasticsearch/migration/postcreate.py
cubicweb_elasticsearch/schema.py
cubicweb_elasticsearch/search_helpers.py
cubicweb_elasticsearch/site_cubicweb.py
cubicweb_elasticsearch/testutils.py
cubicweb_elasticsearch/views.py
debian/compat
debian/control
debian/rules
entities.py
es.py
hooks.py
i18n/en.po
i18n/es.po
i18n/fr.po
migration/postcreate.py
schema.py
search_helpers.py
setup.py
site_cubicweb.py
test/test_compose_search.py
test/test_elastic_search.py
test/test_hooks.py
test/test_parents.py
testutils.py
tox.ini
views.py
# HG changeset patch
# User Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
# Date 1492079018 -7200
# Thu Apr 13 12:23:38 2017 +0200
# Node ID e6392f5058c418affb1b40649c9460ced29f1d61
# Parent 14a2e29584cdb21563bf5b0a1c9891b2d9afffee
[pkg] make elasticsearch a new-layout-style cube

(closes #17071989)

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,5 +1,4 @@
1  include *.py
2  include */*.py
3 -recursive-include data *.gif *.png *.ico *.css *.js
4 -recursive-include i18n *.po
5 -recursive-include wdoc *
6 +recursive-include cubicweb_elasticsearch/data *.gif *.png *.ico *.css *.js
7 +recursive-include cubicweb_elasticsearch/i18n *.po
diff --git a/cubicweb-elasticsearch.spec b/cubicweb-elasticsearch.spec
@@ -32,16 +32,15 @@
8  # change the python version in shebangs
9  find . -name '*.py' -type f -print0 |  xargs -0 sed -i '1,3s;^#!.*python.*$;#! /usr/bin/python2.6;'
10  %endif
11 
12  %install
13 -NO_SETUPTOOLS=1 %{__python} setup.py --quiet install --no-compile --prefix=%{_prefix} --root="$RPM_BUILD_ROOT"
14 +%{__python} setup.py --quiet install --no-compile --prefix=%{_prefix} --root="$RPM_BUILD_ROOT"
15  # remove generated .egg-info file
16  rm -rf $RPM_BUILD_ROOT/usr/lib/python*
17 
18 
19  %clean
20  rm -rf $RPM_BUILD_ROOT
21 
22  %files
23  %defattr(-, root, root)
24 -%{_prefix}/share/cubicweb/cubes/*
diff --git a/__init__.py b/cubicweb_elasticsearch/__init__.py
diff --git a/__pkginfo__.py b/cubicweb_elasticsearch/__pkginfo__.py
@@ -25,30 +25,5 @@
25      'Environment :: Web Environment',
26      'Framework :: CubicWeb',
27      'Programming Language :: Python',
28      'Programming Language :: JavaScript',
29      ]
30 -
31 -from os import listdir as _listdir
32 -from os.path import join, isdir
33 -from glob import glob
34 -
35 -THIS_CUBE_DIR = join('share', 'cubicweb', 'cubes', modname)
36 -
37 -
38 -def listdir(dirpath):
39 -    return [join(dirpath, fname) for fname in _listdir(dirpath)
40 -            if fname[0] != '.' and not fname.endswith('.pyc')
41 -            and not fname.endswith('~')
42 -            and not isdir(join(dirpath, fname))]
43 -
44 -data_files = [
45 -    # common files
46 -    [THIS_CUBE_DIR, [fname for fname in glob('*.py') if fname != 'setup.py']],
47 -    ]
48 -# check for possible extended cube layout
49 -for dname in ('entities', 'views', 'sobjects', 'hooks', 'schema', 'data',
50 -              'wdoc', 'i18n', 'migration'):
51 -    if isdir(dname):
52 -        data_files.append([join(THIS_CUBE_DIR, dname), listdir(dname)])
53 -# Note: here, you'll need to add subdirectories if you want
54 -# them to be included in the debian package
diff --git a/ccplugin.py b/cubicweb_elasticsearch/ccplugin.py
@@ -13,11 +13,11 @@
55 
56  from cubicweb.cwctl import CWCTL, init_cmdline_log_threshold
57  from cubicweb.cwconfig import CubicWebConfiguration as cwcfg
58  from cubicweb.toolsutils import Command
59 
60 -from cubes.elasticsearch.es import indexable_types, fulltext_indexable_rql
61 +from cubicweb_elasticsearch.es import indexable_types, fulltext_indexable_rql
62 
63 
64  HERE = osp.dirname(osp.abspath(__file__))
65 
66 
diff --git a/entities.py b/cubicweb_elasticsearch/entities.py
@@ -22,11 +22,11 @@
67  from cubicweb import view, neg_role
68  from cubicweb.predicates import is_instance
69 
70  from cubicweb.appobject import AppObject
71 
72 -from cubes.elasticsearch import es
73 +from cubicweb_elasticsearch import es
74 
75 
76  def deep_update(d1, d2):
77      for key, value in d2.iteritems():
78          if isinstance(value, collections.Mapping):
diff --git a/es.py b/cubicweb_elasticsearch/es.py
diff --git a/hooks.py b/cubicweb_elasticsearch/hooks.py
@@ -22,11 +22,11 @@
79  from elasticsearch.exceptions import ConnectionError
80  from urllib3.exceptions import ProtocolError
81 
82  from cubicweb.server import hook
83  from cubicweb.predicates import score_entity
84 -from cubes.elasticsearch.es import indexable_types, fulltext_indexable_rql, CUSTOM_ATTRIBUTES
85 +from cubicweb_elasticsearch.es import indexable_types, fulltext_indexable_rql, CUSTOM_ATTRIBUTES
86 
87  log = logging.getLogger(__name__)
88 
89 
90  def entity_indexable(entity):
diff --git a/i18n/en.po b/cubicweb_elasticsearch/i18n/en.po
diff --git a/i18n/es.po b/cubicweb_elasticsearch/i18n/es.po
diff --git a/i18n/fr.po b/cubicweb_elasticsearch/i18n/fr.po
diff --git a/migration/postcreate.py b/cubicweb_elasticsearch/migration/postcreate.py
diff --git a/schema.py b/cubicweb_elasticsearch/schema.py
diff --git a/search_helpers.py b/cubicweb_elasticsearch/search_helpers.py
diff --git a/site_cubicweb.py b/cubicweb_elasticsearch/site_cubicweb.py
diff --git a/testutils.py b/cubicweb_elasticsearch/testutils.py
@@ -3,12 +3,12 @@
91 
92  from elasticsearch_dsl.connections import connections
93 
94  from cubicweb.predicates import is_instance
95 
96 -from cubes.elasticsearch.es import CUSTOM_ATTRIBUTES
97 -from cubes.elasticsearch.entities import IFullTextIndexSerializable
98 +from cubicweb_elasticsearch.es import CUSTOM_ATTRIBUTES
99 +from cubicweb_elasticsearch.entities import IFullTextIndexSerializable
100 
101 
102  CUSTOM_ATTRIBUTES['Blog'] = ('title',)
103 
104 
diff --git a/views.py b/cubicweb_elasticsearch/views.py
@@ -26,12 +26,12 @@
105 
106  import cwtags.tag as t
107  from cubicweb import _
108  from cubicweb.view import StartupView
109 
110 -from cubes.elasticsearch.es import indexable_types, get_connection
111 -from cubes.elasticsearch.search_helpers import compose_search
112 +from cubicweb_elasticsearch.es import indexable_types, get_connection
113 +from cubicweb_elasticsearch.search_helpers import compose_search
114 
115 
116  class CWFacetedSearch(FacetedSearch):
117      # fields that should be searched
118      fields = ["title^3", "description^2", '_all']
diff --git a/debian/compat b/debian/compat
@@ -1,1 +1,1 @@
119 -7
120 +9
diff --git a/debian/control b/debian/control
@@ -1,22 +1,22 @@
121  Source: cubicweb-elasticsearch
122  Section: web
123  Priority: optional
124  Maintainer: LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
125  Build-Depends:
126 - debhelper (>= 7),
127 + debhelper (>= 9),
128   python (>= 2.6.5),
129 -Standards-Version: 3.9.3
130 +Standards-Version: 3.9.6
131  X-Python-Version: >= 2.6
132 
133  Package: cubicweb-elasticsearch
134  Architecture: all
135  Depends:
136 + ${python:Depends},
137 + ${misc:Depends},
138   cubicweb-common (>= 3.22.2),
139   python-six (>= 1.4.0),
140 - ${python:Depends},
141 - ${misc:Depends},
142   python-bs4,
143   python-cwtags,
144   python-elasticsearch,
145   python-elasticsearch-dsl,
146  Description: Simple ElasticSearch indexing integration for CubicWeb
diff --git a/debian/rules b/debian/rules
@@ -1,9 +1,4 @@
147  #!/usr/bin/make -f
148 
149 -export NO_SETUPTOOLS=1
150 -
151  %:
152  	dh $@ --with python2
153 -
154 -override_dh_python2:
155 -	dh_python2 -i /usr/share/cubicweb
diff --git a/setup.py b/setup.py
@@ -1,13 +1,8 @@
156  #!/usr/bin/env python
157  # pylint: disable=W0142,W0403,W0404,W0613,W0622,W0622,W0704,R0904,C0103,E0611
158  #
159 -# copyright 2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
160 -# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
161 -#
162 -# This file is part of CubicWeb.
163 -#
164  # CubicWeb is free software: you can redistribute it and/or modify it under the
165  # terms of the GNU Lesser General Public License as published by the Free
166  # Software Foundation, either version 2.1 of the License, or (at your option)
167  # any later version.
168  #
@@ -16,191 +11,67 @@
169  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
170  # details.
171  #
172  # You should have received a copy of the GNU Lesser General Public License
173  # along with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
174 -"""Generic Setup script, takes package info from __pkginfo__.py file
175 +"""cubicweb_elasticsearch setup module using data from
176 +cubicweb_elasticsearch/__pkginfo__.py file
177  """
178 -__docformat__ = "restructuredtext en"
179 -
180 -import os
181 -import sys
182 -import shutil
183 -from os.path import exists, join, walk
184 -
185 -try:
186 -    if os.environ.get('NO_SETUPTOOLS'):
187 -        raise ImportError()  # do as there is no setuptools
188 -    from setuptools import setup
189 -    from setuptools.command import install_lib
190 -    USE_SETUPTOOLS = True
191 -except ImportError:
192 -    from distutils.core import setup
193 -    from distutils.command import install_lib
194 -    USE_SETUPTOOLS = False
195 -from distutils.command import install_data
196 -
197 -# import required features
198 -from __pkginfo__ import modname, version, license, description, web, \
199 -    author, author_email, classifiers
200 -
201 -if exists('README'):
202 -    long_description = open('README').read()
203 -else:
204 -    long_description = ''
205 
206 -# import optional features
207 -import __pkginfo__
208 -if USE_SETUPTOOLS:
209 -    requires = {}
210 -    for entry in ("__depends__",):  # "__recommends__"):
211 -        requires.update(getattr(__pkginfo__, entry, {}))
212 -    install_requires = [("%s %s" % (d, v and v or "")).strip()
213 -                        for d, v in requires.items()]
214 -else:
215 -    install_requires = []
216 +from os.path import join, dirname, exists
217 +from distutils.command import build
218 
219 -distname = getattr(__pkginfo__, 'distname', modname)
220 -scripts = getattr(__pkginfo__, 'scripts', ())
221 -include_dirs = getattr(__pkginfo__, 'include_dirs', ())
222 -data_files = getattr(__pkginfo__, 'data_files', None)
223 -ext_modules = getattr(__pkginfo__, 'ext_modules', None)
224 -dependency_links = getattr(__pkginfo__, 'dependency_links', ())
225 -
226 -BASE_BLACKLIST = ('CVS', '.svn', '.hg', 'debian', 'dist', 'build')
227 -IGNORED_EXTENSIONS = ('.pyc', '.pyo', '.elc', '~')
228 -
229 -
230 -def ensure_scripts(linux_scripts):
231 -    """
232 -    Creates the proper script names required for each platform
233 -    (taken from 4Suite)
234 -    """
235 -    from distutils import util
236 -    if util.get_platform()[:3] == 'win':
237 -        scripts_ = [script + '.bat' for script in linux_scripts]
238 -    else:
239 -        scripts_ = linux_scripts
240 -    return scripts_
241 +from setuptools import find_packages, setup
242 
243 
244 -def export(from_dir, to_dir,
245 -           blacklist=BASE_BLACKLIST,
246 -           ignore_ext=IGNORED_EXTENSIONS,
247 -           verbose=True):
248 -    """make a mirror of from_dir in to_dir, omitting directories and files
249 -    listed in the black list
250 -    """
251 -    def make_mirror(arg, directory, fnames):
252 -        """walk handler"""
253 -        for norecurs in blacklist:
254 -            try:
255 -                fnames.remove(norecurs)
256 -            except ValueError:
257 -                pass
258 -        for filename in fnames:
259 -            # don't include binary files
260 -            if filename[-4:] in ignore_ext:
261 -                continue
262 -            if filename[-1] == '~':
263 -                continue
264 -            src = join(directory, filename)
265 -            dest = to_dir + src[len(from_dir):]
266 -            if verbose:
267 -                sys.stderr.write('%s -> %s\n' % (src, dest))
268 -            if os.path.isdir(src):
269 -                if not exists(dest):
270 -                    os.mkdir(dest)
271 -            else:
272 -                if exists(dest):
273 -                    os.remove(dest)
274 -                shutil.copy2(src, dest)
275 -    try:
276 -        os.mkdir(to_dir)
277 -    except OSError as ex:
278 -        # file exists ?
279 -        import errno
280 -        if ex.errno != errno.EEXIST:
281 -            raise
282 -    walk(from_dir, make_mirror, None)
283 +here = dirname(__file__)
284 +
285 +# load metadata from the __pkginfo__.py file so there is no risk of conflict
286 +# see https://packaging.python.org/en/latest/single_source_version.html
287 +pkginfo = join(here, 'cubicweb_elasticsearch', '__pkginfo__.py')
288 +__pkginfo__ = {}
289 +with open(pkginfo) as f:
290 +    exec(f.read(), __pkginfo__)
291 +
292 +# get required metadatas
293 +distname = __pkginfo__['distname']
294 +version = __pkginfo__['version']
295 +license = __pkginfo__['license']
296 +description = __pkginfo__['description']
297 +web = __pkginfo__['web']
298 +author = __pkginfo__['author']
299 +author_email = __pkginfo__['author_email']
300 +classifiers = __pkginfo__['classifiers']
301 +
302 +with open(join(here, 'README')) as f:
303 +    long_description = f.read()
304 +
305 +# get optional metadatas
306 +dependency_links = __pkginfo__.get('dependency_links', ())
307 +
308 +requires = {}
309 +for entry in ("__depends__",):  # "__recommends__"):
310 +    requires.update(__pkginfo__.get(entry, {}))
311 +install_requires = ["{0} {1}".format(d, v and v or "").strip()
312 +                    for d, v in requires.items()]
313 
314 
315 -class MyInstallLib(install_lib.install_lib):
316 -    """extend install_lib command to handle  package __init__.py and
317 -    include_dirs variable if necessary
318 -    """
319 -    def run(self):
320 -        """overridden from install_lib class"""
321 -        install_lib.install_lib.run(self)
322 -        # manually install included directories if any
323 -        if include_dirs:
324 -            base = modname
325 -            for directory in include_dirs:
326 -                dest = join(self.install_dir, base, directory)
327 -                export(directory, dest, verbose=False)
328 -
329 -# re-enable copying data files in sys.prefix
330 -old_install_data = install_data.install_data
331 -if USE_SETUPTOOLS:
332 -    # overwrite InstallData to use sys.prefix instead of the egg directory
333 -    class MyInstallData(old_install_data):
334 -        """A class that manages data files installation"""
335 -        def run(self):
336 -            _old_install_dir = self.install_dir
337 -            if self.install_dir.endswith('egg'):
338 -                self.install_dir = sys.prefix
339 -            old_install_data.run(self)
340 -            self.install_dir = _old_install_dir
341 -    try:
342 -        # only if easy_install available
343 -        import setuptools.command.easy_install  # noqa
344 -        # monkey patch: Crack SandboxViolation verification
345 -        from setuptools.sandbox import DirectorySandbox as DS
346 -        old_ok = DS._ok
347 -
348 -        def _ok(self, path):
349 -            """Return True if ``path`` can be written during installation."""
350 -            out = old_ok(self, path)  # here for side effect from setuptools
351 -            realpath = os.path.normcase(os.path.realpath(path))
352 -            allowed_path = os.path.normcase(sys.prefix)
353 -            if realpath.startswith(allowed_path):
354 -                out = True
355 -            return out
356 -        DS._ok = _ok
357 -    except ImportError:
358 -        pass
359 -
360 -
361 -def install(**kwargs):
362 -    """setup entry point"""
363 -    if USE_SETUPTOOLS:
364 -        if '--force-manifest' in sys.argv:
365 -            sys.argv.remove('--force-manifest')
366 -    # install-layout option was introduced in 2.5.3-1~exp1
367 -    elif sys.version_info < (2, 5, 4) and '--install-layout=deb' in sys.argv:
368 -        sys.argv.remove('--install-layout=deb')
369 -    cmdclass = {'install_lib': MyInstallLib}
370 -    if USE_SETUPTOOLS:
371 -        kwargs['install_requires'] = install_requires
372 -        kwargs['dependency_links'] = dependency_links
373 -        kwargs['zip_safe'] = False
374 -        cmdclass['install_data'] = MyInstallData
375 -
376 -    return setup(name=distname,
377 -                 version=version,
378 -                 license=license,
379 -                 description=description,
380 -                 long_description=long_description,
381 -                 author=author,
382 -                 author_email=author_email,
383 -                 url=web,
384 -                 scripts=ensure_scripts(scripts),
385 -                 data_files=data_files,
386 -                 ext_modules=ext_modules,
387 -                 cmdclass=cmdclass,
388 -                 classifiers=classifiers,
389 -                 **kwargs
390 -                 )
391 -
392 -
393 -if __name__ == '__main__':
394 -    install()
395 +setup(
396 +    name=distname,
397 +    version=version,
398 +    license=license,
399 +    description=description,
400 +    long_description=long_description,
401 +    author=author,
402 +    author_email=author_email,
403 +    url=web,
404 +    classifiers=classifiers,
405 +    packages=find_packages(exclude=['test']),
406 +    install_requires=install_requires,
407 +    include_package_data=True,
408 +    entry_points={
409 +        'cubicweb.cubes': [
410 +            'elasticsearch=cubicweb_elasticsearch',
411 +        ]
412 +    },
413 +    zip_safe=False,
414 +)
diff --git a/test/test_compose_search.py b/test/test_compose_search.py
@@ -16,11 +16,11 @@
415 
416  import unittest
417  from elasticsearch_dsl import Search
418 
419  from cubicweb.devtools import testlib
420 -from cubes.elasticsearch.search_helpers import compose_search
421 +from cubicweb_elasticsearch.search_helpers import compose_search
422  from logilab.mtconverter import xml_escape
423 
424 
425  class ComposeSearchTestCase(testlib.TestCase):
426 
diff --git a/test/test_elastic_search.py b/test/test_elastic_search.py
@@ -9,12 +9,12 @@
427 
428  from elasticsearch_dsl.faceted_search import FacetedResponse
429 
430  from cubicweb.devtools import testlib
431  from cubicweb.cwconfig import CubicWebConfiguration
432 -from cubes.elasticsearch import ccplugin
433 -from cubes.elasticsearch.es import (indexable_types,
434 +from cubicweb_elasticsearch import ccplugin
435 +from cubicweb_elasticsearch.es import (indexable_types,
436                                      fulltext_indexable_rql)
437 
438 
439  # TODO - find a way to configure ElasticSearch as non threaded while running tests
440  # so that the traces show the full stack, not just starting from connection.http_*
diff --git a/test/test_hooks.py b/test/test_hooks.py
@@ -3,12 +3,12 @@
441 
442  from elasticsearch_dsl import Search
443 
444  from cubicweb.devtools import testlib
445 
446 -from cubes.elasticsearch.testutils import RealESTestMixin, BlogFTIAdapter
447 -from cubes.elasticsearch.search_helpers import compose_search
448 +from cubicweb_elasticsearch.testutils import RealESTestMixin, BlogFTIAdapter
449 +from cubicweb_elasticsearch.search_helpers import compose_search
450 
451 
452  class ReindexOnRelationTests(RealESTestMixin, testlib.CubicWebTC):
453 
454      def test_es_hooks_modify_relation(self):
diff --git a/test/test_parents.py b/test/test_parents.py
@@ -5,13 +5,13 @@
455 
456  from elasticsearch_dsl import Search
457 
458  from cubicweb.devtools import testlib
459 
460 -from cubes.elasticsearch.search_helpers import compose_search
461 +from cubicweb_elasticsearch.search_helpers import compose_search
462 
463 -from cubes.elasticsearch.testutils import RealESTestMixin, BlogFTIAdapter
464 +from cubicweb_elasticsearch.testutils import RealESTestMixin, BlogFTIAdapter
465 
466 
467  class ParentsSearchTC(RealESTestMixin, testlib.CubicWebTC):
468 
469      def test_parent_search(self):
diff --git a/tox.ini b/tox.ini
@@ -30,6 +30,6 @@
470 
471  [flake8]
472  format = pylint
473  max-line-length = 100
474  ignore = E731,W503
475 -exclude = __pkginfo__.py,migration/*,test/data/*,setup.py,.tox/*
476 +exclude = cubicweb_elasticsearch/__pkginfo__.py,cubicweb_elasticsearch/migration/*,test/data/*,setup.py,.tox/*