[pkg] make elasticsearch a new-layout-style cube

authorAdrien Di Mascio <Adrien.DiMascio@logilab.fr>
changeset60aef4f01192
branchdefault
phasedraft
hiddenyes
parent revision#e6aadade8a9d Added tag 0.4.0, centos/0.4.0-1, debian/0.4.0-1 for changeset 47cbbcc45617
child revision<not specified>
files modified by this revision
MANIFEST.in
__init__.py
__pkginfo__.py
ccplugin.py
cubicweb_elasticsearch/__init__.py
cubicweb_elasticsearch/__pkginfo__.py
cubicweb_elasticsearch/ccplugin.py
cubicweb_elasticsearch/entities.py
cubicweb_elasticsearch/es.py
cubicweb_elasticsearch/hooks.py
cubicweb_elasticsearch/i18n/en.po
cubicweb_elasticsearch/i18n/es.po
cubicweb_elasticsearch/i18n/fr.po
cubicweb_elasticsearch/migration/postcreate.py
cubicweb_elasticsearch/schema.py
cubicweb_elasticsearch/search_helpers.py
cubicweb_elasticsearch/site_cubicweb.py
cubicweb_elasticsearch/testutils.py
cubicweb_elasticsearch/views.py
debian/compat
debian/control
debian/rules
entities.py
es.py
hooks.py
i18n/en.po
i18n/es.po
i18n/fr.po
migration/postcreate.py
schema.py
search_helpers.py
setup.py
site_cubicweb.py
testutils.py
tox.ini
views.py
# HG changeset patch
# User Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
# Date 1492079018 -7200
# Thu Apr 13 12:23:38 2017 +0200
# Node ID 60aef4f01192e1104ec7404d3a3afe80eb70ec2c
# Parent e6aadade8a9d9c01a78a04c05b36f6c099f08ace
[pkg] make elasticsearch a new-layout-style cube

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,5 +1,4 @@
1  include *.py
2  include */*.py
3 -recursive-include data *.gif *.png *.ico *.css *.js
4 -recursive-include i18n *.po
5 -recursive-include wdoc *
6 +recursive-include cubicweb_elasticsearch/data *.gif *.png *.ico *.css *.js
7 +recursive-include cubicweb_elasticsearch/i18n *.po
diff --git a/__init__.py b/cubicweb_elasticsearch/__init__.py
diff --git a/__pkginfo__.py b/cubicweb_elasticsearch/__pkginfo__.py
@@ -25,30 +25,5 @@
8      'Environment :: Web Environment',
9      'Framework :: CubicWeb',
10      'Programming Language :: Python',
11      'Programming Language :: JavaScript',
12      ]
13 -
14 -from os import listdir as _listdir
15 -from os.path import join, isdir
16 -from glob import glob
17 -
18 -THIS_CUBE_DIR = join('share', 'cubicweb', 'cubes', modname)
19 -
20 -
21 -def listdir(dirpath):
22 -    return [join(dirpath, fname) for fname in _listdir(dirpath)
23 -            if fname[0] != '.' and not fname.endswith('.pyc')
24 -            and not fname.endswith('~')
25 -            and not isdir(join(dirpath, fname))]
26 -
27 -data_files = [
28 -    # common files
29 -    [THIS_CUBE_DIR, [fname for fname in glob('*.py') if fname != 'setup.py']],
30 -    ]
31 -# check for possible extended cube layout
32 -for dname in ('entities', 'views', 'sobjects', 'hooks', 'schema', 'data',
33 -              'wdoc', 'i18n', 'migration'):
34 -    if isdir(dname):
35 -        data_files.append([join(THIS_CUBE_DIR, dname), listdir(dname)])
36 -# Note: here, you'll need to add subdirectories if you want
37 -# them to be included in the debian package
diff --git a/ccplugin.py b/cubicweb_elasticsearch/ccplugin.py
@@ -13,11 +13,11 @@
38 
39  from cubicweb.cwctl import CWCTL, init_cmdline_log_threshold
40  from cubicweb.cwconfig import CubicWebConfiguration as cwcfg
41  from cubicweb.toolsutils import Command
42 
43 -from cubes.elasticsearch.es import indexable_types, fulltext_indexable_rql
44 +from cubicweb_elasticsearch.es import indexable_types, fulltext_indexable_rql
45 
46 
47  HERE = osp.dirname(osp.abspath(__file__))
48 
49 
diff --git a/entities.py b/cubicweb_elasticsearch/entities.py
@@ -22,11 +22,11 @@
50  from cubicweb import view, neg_role
51  from cubicweb.predicates import is_instance
52 
53  from cubicweb.appobject import AppObject
54 
55 -from cubes.elasticsearch import es
56 +from cubicweb_elasticsearch import es
57 
58 
59  def deep_update(d1, d2):
60      for key, value in d2.iteritems():
61          if isinstance(value, collections.Mapping):
diff --git a/es.py b/cubicweb_elasticsearch/es.py
diff --git a/hooks.py b/cubicweb_elasticsearch/hooks.py
@@ -22,11 +22,11 @@
62  from elasticsearch.exceptions import ConnectionError
63  from urllib3.exceptions import ProtocolError
64 
65  from cubicweb.server import hook
66  from cubicweb.predicates import score_entity
67 -from cubes.elasticsearch.es import indexable_types, fulltext_indexable_rql, CUSTOM_ATTRIBUTES
68 +from cubicweb_elasticsearch.es import indexable_types, fulltext_indexable_rql, CUSTOM_ATTRIBUTES
69 
70  log = logging.getLogger(__name__)
71 
72 
73  def entity_indexable(entity):
diff --git a/i18n/en.po b/cubicweb_elasticsearch/i18n/en.po
diff --git a/i18n/es.po b/cubicweb_elasticsearch/i18n/es.po
diff --git a/i18n/fr.po b/cubicweb_elasticsearch/i18n/fr.po
diff --git a/migration/postcreate.py b/cubicweb_elasticsearch/migration/postcreate.py
diff --git a/schema.py b/cubicweb_elasticsearch/schema.py
diff --git a/search_helpers.py b/cubicweb_elasticsearch/search_helpers.py
diff --git a/site_cubicweb.py b/cubicweb_elasticsearch/site_cubicweb.py
diff --git a/testutils.py b/cubicweb_elasticsearch/testutils.py
@@ -3,12 +3,12 @@
74 
75  from elasticsearch_dsl.connections import connections
76 
77  from cubicweb.predicates import is_instance
78 
79 -from cubes.elasticsearch.es import CUSTOM_ATTRIBUTES
80 -from cubes.elasticsearch.entities import IFullTextIndexSerializable
81 +from cubicweb_elasticsearch.es import CUSTOM_ATTRIBUTES
82 +from cubicweb_elasticsearch.entities import IFullTextIndexSerializable
83 
84 
85  CUSTOM_ATTRIBUTES['Blog'] = ('title',)
86 
87 
diff --git a/views.py b/cubicweb_elasticsearch/views.py
@@ -26,12 +26,12 @@
88 
89  import cwtags.tag as t
90  from cubicweb import _
91  from cubicweb.view import StartupView
92 
93 -from cubes.elasticsearch.es import indexable_types, get_connection
94 -from cubes.elasticsearch.search_helpers import compose_search
95 +from cubicweb_elasticsearch.es import indexable_types, get_connection
96 +from cubicweb_elasticsearch.search_helpers import compose_search
97 
98 
99  class CWFacetedSearch(FacetedSearch):
100      # fields that should be searched
101      fields = ["title^3", "description^2", '_all']
diff --git a/debian/compat b/debian/compat
@@ -1,1 +1,1 @@
102 -7
103 +9
diff --git a/debian/control b/debian/control
@@ -1,22 +1,22 @@
104  Source: cubicweb-elasticsearch
105  Section: web
106  Priority: optional
107  Maintainer: LOGILAB S.A. (Paris, FRANCE) <contact@logilab.fr>
108  Build-Depends:
109 - debhelper (>= 7),
110 + debhelper (>= 9),
111   python (>= 2.6.5),
112 -Standards-Version: 3.9.3
113 +Standards-Version: 3.9.6
114  X-Python-Version: >= 2.6
115 
116  Package: cubicweb-elasticsearch
117  Architecture: all
118  Depends:
119 + ${python:Depends},
120 + ${misc:Depends},
121   cubicweb-common (>= 3.22.2),
122   python-six (>= 1.4.0),
123 - ${python:Depends},
124 - ${misc:Depends},
125   python-bs4,
126   python-cwtags,
127   python-elasticsearch,
128   python-elasticsearch-dsl,
129  Description: Simple ElasticSearch indexing integration for CubicWeb
diff --git a/debian/rules b/debian/rules
@@ -1,9 +1,4 @@
130  #!/usr/bin/make -f
131 
132 -export NO_SETUPTOOLS=1
133 -
134  %:
135  	dh $@ --with python2
136 -
137 -override_dh_python2:
138 -	dh_python2 -i /usr/share/cubicweb
diff --git a/setup.py b/setup.py
@@ -1,13 +1,8 @@
139  #!/usr/bin/env python
140  # pylint: disable=W0142,W0403,W0404,W0613,W0622,W0622,W0704,R0904,C0103,E0611
141  #
142 -# copyright 2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
143 -# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
144 -#
145 -# This file is part of CubicWeb.
146 -#
147  # CubicWeb is free software: you can redistribute it and/or modify it under the
148  # terms of the GNU Lesser General Public License as published by the Free
149  # Software Foundation, either version 2.1 of the License, or (at your option)
150  # any later version.
151  #
@@ -16,191 +11,67 @@
152  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
153  # details.
154  #
155  # You should have received a copy of the GNU Lesser General Public License
156  # along with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
157 -"""Generic Setup script, takes package info from __pkginfo__.py file
158 +"""cubicweb_elasticsearch setup module using data from
159 +cubicweb_elasticsearch/__pkginfo__.py file
160  """
161 -__docformat__ = "restructuredtext en"
162 -
163 -import os
164 -import sys
165 -import shutil
166 -from os.path import exists, join, walk
167 -
168 -try:
169 -    if os.environ.get('NO_SETUPTOOLS'):
170 -        raise ImportError()  # do as there is no setuptools
171 -    from setuptools import setup
172 -    from setuptools.command import install_lib
173 -    USE_SETUPTOOLS = True
174 -except ImportError:
175 -    from distutils.core import setup
176 -    from distutils.command import install_lib
177 -    USE_SETUPTOOLS = False
178 -from distutils.command import install_data
179 -
180 -# import required features
181 -from __pkginfo__ import modname, version, license, description, web, \
182 -    author, author_email, classifiers
183 -
184 -if exists('README'):
185 -    long_description = open('README').read()
186 -else:
187 -    long_description = ''
188 
189 -# import optional features
190 -import __pkginfo__
191 -if USE_SETUPTOOLS:
192 -    requires = {}
193 -    for entry in ("__depends__",):  # "__recommends__"):
194 -        requires.update(getattr(__pkginfo__, entry, {}))
195 -    install_requires = [("%s %s" % (d, v and v or "")).strip()
196 -                        for d, v in requires.items()]
197 -else:
198 -    install_requires = []
199 +from os.path import join, dirname, exists
200 +from distutils.command import build
201 
202 -distname = getattr(__pkginfo__, 'distname', modname)
203 -scripts = getattr(__pkginfo__, 'scripts', ())
204 -include_dirs = getattr(__pkginfo__, 'include_dirs', ())
205 -data_files = getattr(__pkginfo__, 'data_files', None)
206 -ext_modules = getattr(__pkginfo__, 'ext_modules', None)
207 -dependency_links = getattr(__pkginfo__, 'dependency_links', ())
208 -
209 -BASE_BLACKLIST = ('CVS', '.svn', '.hg', 'debian', 'dist', 'build')
210 -IGNORED_EXTENSIONS = ('.pyc', '.pyo', '.elc', '~')
211 -
212 -
213 -def ensure_scripts(linux_scripts):
214 -    """
215 -    Creates the proper script names required for each platform
216 -    (taken from 4Suite)
217 -    """
218 -    from distutils import util
219 -    if util.get_platform()[:3] == 'win':
220 -        scripts_ = [script + '.bat' for script in linux_scripts]
221 -    else:
222 -        scripts_ = linux_scripts
223 -    return scripts_
224 +from setuptools import find_packages, setup
225 
226 
227 -def export(from_dir, to_dir,
228 -           blacklist=BASE_BLACKLIST,
229 -           ignore_ext=IGNORED_EXTENSIONS,
230 -           verbose=True):
231 -    """make a mirror of from_dir in to_dir, omitting directories and files
232 -    listed in the black list
233 -    """
234 -    def make_mirror(arg, directory, fnames):
235 -        """walk handler"""
236 -        for norecurs in blacklist:
237 -            try:
238 -                fnames.remove(norecurs)
239 -            except ValueError:
240 -                pass
241 -        for filename in fnames:
242 -            # don't include binary files
243 -            if filename[-4:] in ignore_ext:
244 -                continue
245 -            if filename[-1] == '~':
246 -                continue
247 -            src = join(directory, filename)
248 -            dest = to_dir + src[len(from_dir):]
249 -            if verbose:
250 -                sys.stderr.write('%s -> %s\n' % (src, dest))
251 -            if os.path.isdir(src):
252 -                if not exists(dest):
253 -                    os.mkdir(dest)
254 -            else:
255 -                if exists(dest):
256 -                    os.remove(dest)
257 -                shutil.copy2(src, dest)
258 -    try:
259 -        os.mkdir(to_dir)
260 -    except OSError as ex:
261 -        # file exists ?
262 -        import errno
263 -        if ex.errno != errno.EEXIST:
264 -            raise
265 -    walk(from_dir, make_mirror, None)
266 +here = dirname(__file__)
267 +
268 +# load metadata from the __pkginfo__.py file so there is no risk of conflict
269 +# see https://packaging.python.org/en/latest/single_source_version.html
270 +pkginfo = join(here, 'cubicweb_elasticsearch', '__pkginfo__.py')
271 +__pkginfo__ = {}
272 +with open(pkginfo) as f:
273 +    exec(f.read(), __pkginfo__)
274 +
275 +# get required metadatas
276 +distname = __pkginfo__['distname']
277 +version = __pkginfo__['version']
278 +license = __pkginfo__['license']
279 +description = __pkginfo__['description']
280 +web = __pkginfo__['web']
281 +author = __pkginfo__['author']
282 +author_email = __pkginfo__['author_email']
283 +classifiers = __pkginfo__['classifiers']
284 +
285 +with open(join(here, 'README')) as f:
286 +    long_description = f.read()
287 +
288 +# get optional metadatas
289 +dependency_links = __pkginfo__.get('dependency_links', ())
290 +
291 +requires = {}
292 +for entry in ("__depends__",):  # "__recommends__"):
293 +    requires.update(__pkginfo__.get(entry, {}))
294 +install_requires = ["{0} {1}".format(d, v and v or "").strip()
295 +                    for d, v in requires.items()]
296 
297 
298 -class MyInstallLib(install_lib.install_lib):
299 -    """extend install_lib command to handle  package __init__.py and
300 -    include_dirs variable if necessary
301 -    """
302 -    def run(self):
303 -        """overridden from install_lib class"""
304 -        install_lib.install_lib.run(self)
305 -        # manually install included directories if any
306 -        if include_dirs:
307 -            base = modname
308 -            for directory in include_dirs:
309 -                dest = join(self.install_dir, base, directory)
310 -                export(directory, dest, verbose=False)
311 -
312 -# re-enable copying data files in sys.prefix
313 -old_install_data = install_data.install_data
314 -if USE_SETUPTOOLS:
315 -    # overwrite InstallData to use sys.prefix instead of the egg directory
316 -    class MyInstallData(old_install_data):
317 -        """A class that manages data files installation"""
318 -        def run(self):
319 -            _old_install_dir = self.install_dir
320 -            if self.install_dir.endswith('egg'):
321 -                self.install_dir = sys.prefix
322 -            old_install_data.run(self)
323 -            self.install_dir = _old_install_dir
324 -    try:
325 -        # only if easy_install available
326 -        import setuptools.command.easy_install  # noqa
327 -        # monkey patch: Crack SandboxViolation verification
328 -        from setuptools.sandbox import DirectorySandbox as DS
329 -        old_ok = DS._ok
330 -
331 -        def _ok(self, path):
332 -            """Return True if ``path`` can be written during installation."""
333 -            out = old_ok(self, path)  # here for side effect from setuptools
334 -            realpath = os.path.normcase(os.path.realpath(path))
335 -            allowed_path = os.path.normcase(sys.prefix)
336 -            if realpath.startswith(allowed_path):
337 -                out = True
338 -            return out
339 -        DS._ok = _ok
340 -    except ImportError:
341 -        pass
342 -
343 -
344 -def install(**kwargs):
345 -    """setup entry point"""
346 -    if USE_SETUPTOOLS:
347 -        if '--force-manifest' in sys.argv:
348 -            sys.argv.remove('--force-manifest')
349 -    # install-layout option was introduced in 2.5.3-1~exp1
350 -    elif sys.version_info < (2, 5, 4) and '--install-layout=deb' in sys.argv:
351 -        sys.argv.remove('--install-layout=deb')
352 -    cmdclass = {'install_lib': MyInstallLib}
353 -    if USE_SETUPTOOLS:
354 -        kwargs['install_requires'] = install_requires
355 -        kwargs['dependency_links'] = dependency_links
356 -        kwargs['zip_safe'] = False
357 -        cmdclass['install_data'] = MyInstallData
358 -
359 -    return setup(name=distname,
360 -                 version=version,
361 -                 license=license,
362 -                 description=description,
363 -                 long_description=long_description,
364 -                 author=author,
365 -                 author_email=author_email,
366 -                 url=web,
367 -                 scripts=ensure_scripts(scripts),
368 -                 data_files=data_files,
369 -                 ext_modules=ext_modules,
370 -                 cmdclass=cmdclass,
371 -                 classifiers=classifiers,
372 -                 **kwargs
373 -                 )
374 -
375 -
376 -if __name__ == '__main__':
377 -    install()
378 +setup(
379 +    name=distname,
380 +    version=version,
381 +    license=license,
382 +    description=description,
383 +    long_description=long_description,
384 +    author=author,
385 +    author_email=author_email,
386 +    url=web,
387 +    classifiers=classifiers,
388 +    packages=find_packages(exclude=['test']),
389 +    install_requires=install_requires,
390 +    include_package_data=True,
391 +    entry_points={
392 +        'cubicweb.cubes': [
393 +            'elasticsearch=cubicweb_elasticsearch',
394 +        ]
395 +    },
396 +    zip_safe=False,
397 +)
diff --git a/tox.ini b/tox.ini
@@ -30,6 +30,6 @@
398 
399  [flake8]
400  format = pylint
401  max-line-length = 100
402  ignore = E731,W503
403 -exclude = __pkginfo__.py,migration/*,test/data/*,setup.py,.tox/*
404 +exclude = cubicweb_elasticsearch/__pkginfo__.py,cubicweb_elasticsearch/migration/*,test/data/*,setup.py,.tox/*