Browse Source

Add a copy of WebOOB modules

Phyks (Lucas Verney) 8 months ago
parent
commit
9a532c0da1

+ 7
- 0
doc/0.getting_started.md View File

@@ -27,6 +27,13 @@ your disk, to point `modules_path` configuration option to
27 27
 `path_to_weboob_git/modules` (see the configuration section below) and to run
28 28
 a `git pull; python setup.py install` in the WebOOB git repo often.
29 29
 
30
+A copy of the WebOOB modules is available in the `modules` directory at the
31
+root of this repository, you can use `"modules_path": "/path/to/flatisfy/modules"` to use them.
32
+This copy may or may not be more up to date than the current state of official
33
+WebOOB modules. Some changes are made there, which are not backported
34
+upstream. WebOOB official modules are not synced in the `modules` folder on a
35
+regular basis, so try both and see which ones match your needs! :)
36
+
30 37
 
31 38
 ## TL;DR
32 39
 

+ 24
- 0
modules/explorimmo/__init__.py View File

@@ -0,0 +1,24 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+
21
+from .module import ExplorimmoModule
22
+
23
+
24
+__all__ = ['ExplorimmoModule']

+ 92
- 0
modules/explorimmo/browser.py View File

@@ -0,0 +1,92 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from weboob.browser import PagesBrowser, URL
21
+from weboob.capabilities.housing import (TypeNotSupported, POSTS_TYPES,
22
+                                         HOUSE_TYPES)
23
+from weboob.tools.compat import urlencode
24
+from .pages import CitiesPage, SearchPage, HousingPage, HousingPage2, PhonePage
25
+
26
+
27
+class ExplorimmoBrowser(PagesBrowser):
28
+    BASEURL = 'https://immobilier.lefigaro.fr'
29
+
30
+    cities = URL('/rest/locations\?q=(?P<city>.*)', CitiesPage)
31
+    search = URL('/annonces/resultat/annonces.html\?(?P<query>.*)', SearchPage)
32
+    housing_html = URL('/annonces/annonce-(?P<_id>.*).html', HousingPage)
33
+    phone = URL('/rest/classifieds/(?P<_id>.*)/phone', PhonePage)
34
+    housing = URL('/rest/classifieds/(?P<_id>.*)',
35
+                  '/rest/classifieds/\?(?P<js_datas>.*)', HousingPage2)
36
+
37
+    TYPES = {POSTS_TYPES.RENT: 'location',
38
+             POSTS_TYPES.SALE: 'vente',
39
+             POSTS_TYPES.FURNISHED_RENT: 'location',
40
+             POSTS_TYPES.VIAGER: 'vente'}
41
+
42
+    RET = {HOUSE_TYPES.HOUSE: 'Maison',
43
+           HOUSE_TYPES.APART: 'Appartement',
44
+           HOUSE_TYPES.LAND: 'Terrain',
45
+           HOUSE_TYPES.PARKING: 'Parking',
46
+           HOUSE_TYPES.OTHER: 'Divers'}
47
+
48
+    def get_cities(self, pattern):
49
+        return self.cities.open(city=pattern).get_cities()
50
+
51
+    def search_housings(self, type, cities, nb_rooms, area_min, area_max,
52
+                        cost_min, cost_max, house_types, advert_types):
53
+
54
+        if type not in self.TYPES:
55
+            raise TypeNotSupported()
56
+
57
+        ret = []
58
+        if type == POSTS_TYPES.VIAGER:
59
+            ret = ['Viager']
60
+        else:
61
+            for house_type in house_types:
62
+                if house_type in self.RET:
63
+                    ret.append(self.RET.get(house_type))
64
+
65
+        data = {'location': ','.join(cities).encode('iso 8859-1'),
66
+                'furnished': type == POSTS_TYPES.FURNISHED_RENT,
67
+                'areaMin': area_min or '',
68
+                'areaMax': area_max or '',
69
+                'priceMin': cost_min or '',
70
+                'priceMax': cost_max or '',
71
+                'transaction': self.TYPES.get(type, 'location'),
72
+                'recherche': '',
73
+                'mode': '',
74
+                'proximity': '0',
75
+                'roomMin': nb_rooms or '',
76
+                'page': '1'}
77
+
78
+        query = u'%s%s%s' % (urlencode(data), '&type=', '&type='.join(ret))
79
+
80
+        return self.search.go(query=query).iter_housings(
81
+            query_type=type,
82
+            advert_types=advert_types
83
+        )
84
+
85
+    def get_housing(self, _id, housing=None):
86
+        return self.housing.go(_id=_id).get_housing(obj=housing)
87
+
88
+    def get_phone(self, _id):
89
+        return self.phone.go(_id=_id).get_phone()
90
+
91
+    def get_total_page(self, js_datas):
92
+        return self.housing.open(js_datas=js_datas).get_total_page()

+ 80
- 0
modules/explorimmo/module.py View File

@@ -0,0 +1,80 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+
21
+from weboob.tools.backend import Module
22
+from weboob.capabilities.housing import CapHousing, Housing, HousingPhoto
23
+
24
+from .browser import ExplorimmoBrowser
25
+
26
+
27
+__all__ = ['ExplorimmoModule']
28
+
29
+
30
+class ExplorimmoModule(Module, CapHousing):
31
+    NAME = 'explorimmo'
32
+    DESCRIPTION = u'explorimmo website'
33
+    MAINTAINER = u'Bezleputh'
34
+    EMAIL = 'carton_ben@yahoo.fr'
35
+    LICENSE = 'AGPLv3+'
36
+    VERSION = '2.1'
37
+
38
+    BROWSER = ExplorimmoBrowser
39
+
40
+    def get_housing(self, housing):
41
+        if isinstance(housing, Housing):
42
+            id = housing.id
43
+        else:
44
+            id = housing
45
+            housing = None
46
+        housing = self.browser.get_housing(id, housing)
47
+        return housing
48
+
49
+    def search_city(self, pattern):
50
+        return self.browser.get_cities(pattern)
51
+
52
+    def search_housings(self, query):
53
+        cities = ['%s' % c.id for c in query.cities if c.backend == self.name]
54
+        if len(cities) == 0:
55
+            return list()
56
+
57
+        return self.browser.search_housings(query.type, cities, query.nb_rooms,
58
+                                            query.area_min, query.area_max,
59
+                                            query.cost_min, query.cost_max,
60
+                                            query.house_types,
61
+                                            query.advert_types)
62
+
63
+    def fill_housing(self, housing, fields):
64
+        if 'phone' in fields:
65
+            housing.phone = self.browser.get_phone(housing.id)
66
+            fields.remove('phone')
67
+
68
+        if len(fields) > 0:
69
+            self.browser.get_housing(housing.id, housing)
70
+
71
+        return housing
72
+
73
+    def fill_photo(self, photo, fields):
74
+        if 'data' in fields and photo.url and not photo.data:
75
+            photo.data = self.browser.open(photo.url).content
76
+        return photo
77
+
78
+    OBJECTS = {Housing: fill_housing,
79
+               HousingPhoto: fill_photo,
80
+               }

+ 455
- 0
modules/explorimmo/pages.py View File

@@ -0,0 +1,455 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+from __future__ import unicode_literals
20
+
21
+import json
22
+import math
23
+import re
24
+from decimal import Decimal
25
+from datetime import datetime
26
+from weboob.browser.filters.json import Dict
27
+from weboob.browser.elements import ItemElement, ListElement, DictElement, method
28
+from weboob.browser.pages import JsonPage, HTMLPage, pagination
29
+from weboob.browser.filters.standard import (CleanText, CleanDecimal, Currency,
30
+                                             Regexp, Env, BrowserURL, Filter,
31
+                                             Format)
32
+from weboob.browser.filters.html import Attr, CleanHTML, XPath
33
+from weboob.capabilities.base import NotAvailable, NotLoaded, Currency as BaseCurrency
34
+from weboob.capabilities.housing import (Housing, HousingPhoto, City,
35
+                                         UTILITIES, ENERGY_CLASS, POSTS_TYPES,
36
+                                         ADVERT_TYPES, HOUSE_TYPES)
37
+from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
38
+from weboob.tools.compat import unquote
39
+
40
+
41
+class CitiesPage(JsonPage):
42
+
43
+    ENCODING = 'UTF-8'
44
+
45
+    def build_doc(self, content):
46
+        content = super(CitiesPage, self).build_doc(content)
47
+        if content:
48
+            return content
49
+        else:
50
+            return [{"locations": []}]
51
+
52
+    @method
53
+    class get_cities(DictElement):
54
+        item_xpath = '0/locations'
55
+
56
+        class item(ItemElement):
57
+            klass = City
58
+
59
+            obj_id = Dict('label')
60
+            obj_name = Dict('label')
61
+
62
+
63
+class SearchPage(HTMLPage):
64
+    @pagination
65
+    @method
66
+    class iter_housings(ListElement):
67
+        item_xpath = '//div[starts-with(@id, "bloc-vue-")]'
68
+
69
+        def next_page(self):
70
+            js_datas = CleanText(
71
+                '//div[@id="js-data"]/@data-rest-search-request'
72
+            )(self).split('?')[-1].split('&')
73
+
74
+            try:
75
+                resultsPerPage = next(
76
+                    x for x in js_datas if 'resultsPerPage' in x
77
+                ).split('=')[-1]
78
+                currentPageNumber = next(
79
+                    x for x in js_datas if 'currentPageNumber' in x
80
+                ).split('=')[-1]
81
+                resultCount = CleanText(
82
+                    '(//div[@id="js-data"]/@data-result-count)[1]'
83
+                )(self)
84
+                totalPageNumber = math.ceil(
85
+                    int(resultCount) / int(resultsPerPage)
86
+                )
87
+
88
+                next_page = int(currentPageNumber) + 1
89
+                if next_page <= totalPageNumber:
90
+                    return self.page.url.replace(
91
+                        'page=%s' % currentPageNumber,
92
+                        'page=%d' % next_page
93
+                    )
94
+            except StopIteration:
95
+                pass
96
+
97
+        class item(ItemElement):
98
+            klass = Housing
99
+            price_selector = './/span[@class="price-label"]|./div/div[@class="item-price-pdf"]'
100
+
101
+            def is_agency(self):
102
+                agency = CleanText('.//span[has-class("item-agency-name")]')(self.el)
103
+                return 'annonce de particulier' not in agency.lower()
104
+
105
+            def condition(self):
106
+                if len(self.env['advert_types']) == 1:
107
+                    is_agency = self.is_agency()
108
+                    if self.env['advert_types'][0] == ADVERT_TYPES.PERSONAL:
109
+                        return not is_agency
110
+                    elif self.env['advert_types'][0] == ADVERT_TYPES.PROFESSIONAL:
111
+                        return is_agency
112
+                return Attr('.', 'data-classified-id', default=False)(self)
113
+
114
+            obj_id = Attr('.', 'data-classified-id')
115
+            obj_type = Env('query_type')
116
+            obj_title = CleanText('./div/h2[@class="item-type"]')
117
+
118
+            def obj_advert_type(self):
119
+                if self.is_agency():
120
+                    return ADVERT_TYPES.PROFESSIONAL
121
+                else:
122
+                    return ADVERT_TYPES.PERSONAL
123
+
124
+            def obj_house_type(self):
125
+                type = self.obj_title(self).split()[0].lower()
126
+                if type == "appartement" or type == "studio" or type == "chambre":
127
+                    return HOUSE_TYPES.APART
128
+                elif type == "maison" or type == "villa":
129
+                    return HOUSE_TYPES.HOUSE
130
+                elif type == "parking":
131
+                    return HOUSE_TYPES.PARKING
132
+                elif type == "terrain":
133
+                    return HOUSE_TYPES.LAND
134
+                else:
135
+                    return HOUSE_TYPES.OTHER
136
+
137
+            def obj_location(self):
138
+                script = CleanText('./script')(self)
139
+                try:
140
+                    # Should be standard JSON+LD data
141
+                    script = json.loads(script)
142
+                except ValueError:
143
+                    try:
144
+                        # But explorimmo can't write JSON correctly and there
145
+                        # is a trailing "}"
146
+                        script = json.loads(script.strip().rstrip('}'))
147
+                    except ValueError:
148
+                        script = None
149
+                if not script:
150
+                    return NotLoaded
151
+
152
+                try:
153
+                    return '%s (%s)' % (
154
+                        script['address']['addressLocality'],
155
+                        script['address']['postalCode']
156
+                    )
157
+                except (KeyError):
158
+                    return NotLoaded
159
+
160
+            def obj_cost(self):
161
+                cost = CleanDecimal(Regexp(CleanText(self.price_selector, default=''),
162
+                                           r'de (.*) à .*',
163
+                                           default=0))(self)
164
+                if cost == 0:
165
+                    return CleanDecimal(self.price_selector, default=NotAvailable)(self)
166
+                else:
167
+                    return cost
168
+
169
+            obj_currency = Currency(price_selector)
170
+
171
+            def obj_utilities(self):
172
+                utilities = CleanText(
173
+                    './div/div/span[@class="price-label"]|'
174
+                    './div/div[@class="item-price-pdf"]|'
175
+                    './div/div/span[@class="item-price"]'
176
+                )(self)
177
+                if "CC" in utilities:
178
+                    return UTILITIES.INCLUDED
179
+                else:
180
+                    return UTILITIES.UNKNOWN
181
+
182
+            obj_text = CleanText('./div/p[@itemprop="description"]')
183
+            obj_area = CleanDecimal(
184
+                Regexp(
185
+                    obj_title,
186
+                    r'(.*?)([\d,\.]*) m2(.*?)',
187
+                    '\\2',
188
+                    default=None
189
+                ),
190
+                replace_dots=True,
191
+                default=NotLoaded
192
+            )
193
+
194
+            obj_url = Format(
195
+                "https://immobilier.lefigaro.fr/annonces/annonce-%s.html",
196
+                CleanText('./@data-classified-id')
197
+            )
198
+
199
+            obj_price_per_meter = PricePerMeterFilter()
200
+
201
+            def obj_phone(self):
202
+                phone = CleanText('./div/div/ul/li[has-class("js-clickphone")]',
203
+                                  replace=[('Téléphoner : ', '')],
204
+                                  default=NotLoaded)(self)
205
+
206
+                if '...' in phone:
207
+                    return NotLoaded
208
+
209
+                return phone
210
+
211
+            def obj_details(self):
212
+                charges = CleanText('.//span[@class="price-fees"]',
213
+                                    default=None)(self)
214
+                if charges:
215
+                    return {
216
+                        "fees": charges.split(":")[1].strip()
217
+                    }
218
+                else:
219
+                    return NotLoaded
220
+
221
+            def obj_photos(self):
222
+                url = CleanText('./div[has-class("default-img")]/img/@data-src')(self)
223
+                if url:
224
+                    url = unquote(url)
225
+                    if "http://" in url[3:]:
226
+                        rindex = url.rfind("?")
227
+                        if rindex == -1:
228
+                            rindex = None
229
+                        url = url[url.find("http://", 3):rindex]
230
+                    return [HousingPhoto(url)]
231
+                else:
232
+                    return NotLoaded
233
+
234
+
235
+class TypeDecimal(Filter):
236
+    def filter(self, el):
237
+        return Decimal(el)
238
+
239
+
240
+class FromTimestamp(Filter):
241
+    def filter(self, el):
242
+        return datetime.fromtimestamp(el / 1000.0)
243
+
244
+
245
+class PhonePage(JsonPage):
246
+    def get_phone(self):
247
+        return self.doc.get('phoneNumber')
248
+
249
+
250
+class HousingPage2(JsonPage):
251
+    @method
252
+    class get_housing(ItemElement):
253
+        klass = Housing
254
+
255
+        def is_agency(self):
256
+            return Dict('agency/isParticulier')(self) == 'false'
257
+
258
+        obj_id = Env('_id')
259
+
260
+        def obj_type(self):
261
+            transaction = Dict('characteristics/transaction')(self)
262
+            if transaction == 'location':
263
+                if Dict('characteristics/isFurnished')(self):
264
+                    return POSTS_TYPES.FURNISHED_RENT
265
+                else:
266
+                    return POSTS_TYPES.RENT
267
+            elif transaction == 'vente':
268
+                type = Dict('characteristics/estateType')(self).lower()
269
+                if 'viager' in type:
270
+                    return POSTS_TYPES.VIAGER
271
+                else:
272
+                    return POSTS_TYPES.SALE
273
+            else:
274
+                return NotAvailable
275
+
276
+        def obj_advert_type(self):
277
+            if self.is_agency:
278
+                return ADVERT_TYPES.PROFESSIONAL
279
+            else:
280
+                return ADVERT_TYPES.PERSONAL
281
+
282
+        def obj_house_type(self):
283
+            type = Dict('characteristics/estateType')(self).lower()
284
+            if 'appartement' in type:
285
+                return HOUSE_TYPES.APART
286
+            elif 'maison' in type:
287
+                return HOUSE_TYPES.HOUSE
288
+            elif 'parking' in type:
289
+                return HOUSE_TYPES.PARKING
290
+            elif 'terrain' in type:
291
+                return HOUSE_TYPES.LAND
292
+            else:
293
+                return HOUSE_TYPES.OTHER
294
+
295
+        obj_title = Dict('characteristics/titleWithTransaction')
296
+        obj_location = Format('%s %s %s', Dict('location/address'),
297
+                              Dict('location/cityLabel'),
298
+                              Dict('location/postalCode'))
299
+
300
+        def obj_cost(self):
301
+            cost = TypeDecimal(Dict('characteristics/price'))(self)
302
+            if cost == 0:
303
+                cost = TypeDecimal(Dict('characteristics/priceMin'))(self)
304
+            return cost
305
+
306
+        obj_currency = BaseCurrency.get_currency('€')
307
+
308
+        def obj_utilities(self):
309
+            are_fees_included = Dict('characteristics/areFeesIncluded',
310
+                                     default=None)(self)
311
+            if are_fees_included:
312
+                return UTILITIES.INCLUDED
313
+            else:
314
+                return UTILITIES.EXCLUDED
315
+
316
+        obj_text = CleanHTML(Dict('characteristics/description'))
317
+        obj_url = BrowserURL('housing_html', _id=Env('_id'))
318
+
319
+        def obj_area(self):
320
+            area = TypeDecimal(Dict('characteristics/area'))(self)
321
+            if area == 0:
322
+                area = TypeDecimal(Dict('characteristics/areaMin'))(self)
323
+            return area
324
+
325
+        obj_date = FromTimestamp(Dict('characteristics/date'))
326
+        obj_bedrooms = TypeDecimal(Dict('characteristics/bedroomCount'))
327
+
328
+        def obj_rooms(self):
329
+            # TODO: Why is roomCount a list?
330
+            rooms = Dict('characteristics/roomCount', default=[])(self)
331
+            if rooms:
332
+                return TypeDecimal(rooms[0])(self)
333
+            return NotAvailable
334
+
335
+        obj_price_per_meter = PricePerMeterFilter()
336
+
337
+        def obj_photos(self):
338
+            photos = []
339
+            for img in Dict('characteristics/images')(self):
340
+                m = re.search('http://thbr\.figarocms\.net.*(http://.*)', img.get('xl'))
341
+                if m:
342
+                    photos.append(HousingPhoto(m.group(1)))
343
+                else:
344
+                    photos.append(HousingPhoto(img.get('xl')))
345
+            return photos
346
+
347
+        def obj_DPE(self):
348
+            DPE = Dict(
349
+                'characteristics/energyConsumptionCategory',
350
+                default=""
351
+            )(self)
352
+            return getattr(ENERGY_CLASS, DPE, NotAvailable)
353
+
354
+        def obj_GES(self):
355
+            GES = Dict(
356
+                'characteristics/greenhouseGasEmissionCategory',
357
+                default=""
358
+            )(self)
359
+            return getattr(ENERGY_CLASS, GES, NotAvailable)
360
+
361
+        def obj_details(self):
362
+            details = {}
363
+            details['fees'] = Dict(
364
+                'characteristics/fees', default=NotAvailable
365
+            )(self)
366
+            details['agencyFees'] = Dict(
367
+                'characteristics/agencyFees', default=NotAvailable
368
+            )(self)
369
+            details['guarantee'] = Dict(
370
+                'characteristics/guarantee', default=NotAvailable
371
+            )(self)
372
+            details['bathrooms'] = Dict(
373
+                'characteristics/bathroomCount', default=NotAvailable
374
+            )(self)
375
+            details['creationDate'] = FromTimestamp(
376
+                                          Dict(
377
+                                              'characteristics/creationDate', default=NotAvailable
378
+                                          ),
379
+                                          default=NotAvailable
380
+            )(self)
381
+            details['availabilityDate'] = Dict(
382
+                'characteristics/estateAvailabilityDate', default=NotAvailable
383
+            )(self)
384
+            details['exposure'] = Dict(
385
+                'characteristics/exposure', default=NotAvailable
386
+            )(self)
387
+            details['heatingType'] = Dict(
388
+                'characteristics/heatingType', default=NotAvailable
389
+            )(self)
390
+            details['floor'] = Dict(
391
+                'characteristics/floor', default=NotAvailable
392
+            )(self)
393
+            details['bedrooms'] = Dict(
394
+                'characteristics/bedroomCount', default=NotAvailable
395
+            )(self)
396
+            details['isFurnished'] = Dict(
397
+                'characteristics/isFurnished', default=NotAvailable
398
+            )(self)
399
+            rooms = Dict('characteristics/roomCount', default=[])(self)
400
+            if len(rooms):
401
+                details['rooms'] = rooms[0]
402
+            details['available'] = Dict(
403
+                'characteristics/isAvailable', default=NotAvailable
404
+            )(self)
405
+            agency = Dict('agency', default=NotAvailable)(self)
406
+            details['agency'] = ', '.join([
407
+                x for x in [
408
+                    agency.get('corporateName', ''),
409
+                    agency.get('corporateAddress', ''),
410
+                    agency.get('corporatePostalCode', ''),
411
+                    agency.get('corporateCity', '')
412
+                ] if x
413
+            ])
414
+            return details
415
+
416
+    def get_total_page(self):
417
+        return self.doc.get('pagination').get('total') if 'pagination' in self.doc else 0
418
+
419
+
420
+class HousingPage(HTMLPage):
421
+    @method
422
+    class get_housing(ItemElement):
423
+        klass = Housing
424
+
425
+        obj_id = Env('_id')
426
+        obj_title = CleanText('//h1[@itemprop="name"]')
427
+        obj_location = CleanText('//span[@class="informations-localisation"]')
428
+        obj_cost = CleanDecimal('//span[@itemprop="price"]')
429
+        obj_currency = Currency('//span[@itemprop="price"]')
430
+        obj_text = CleanHTML('//div[@itemprop="description"]')
431
+        obj_url = BrowserURL('housing', _id=Env('_id'))
432
+        obj_area = CleanDecimal(Regexp(CleanText('//h1[@itemprop="name"]'),
433
+                                       r'(.*?)(\d*) m2(.*?)', '\\2'), default=NotAvailable)
434
+        obj_price_per_meter = PricePerMeterFilter()
435
+
436
+        def obj_photos(self):
437
+            photos = []
438
+            for img in XPath('//a[@class="thumbnail-link"]/img[@itemprop="image"]')(self):
439
+                url = Regexp(CleanText('./@src'), r'http://thbr\.figarocms\.net.*(http://.*)')(img)
440
+                photos.append(HousingPhoto(url))
441
+            return photos
442
+
443
+        def obj_details(self):
444
+            details = dict()
445
+            for item in XPath('//div[@class="features clearfix"]/ul/li')(self):
446
+                key = CleanText('./span[@class="name"]')(item)
447
+                value = CleanText('./span[@class="value"]')(item)
448
+                if value and key:
449
+                    details[key] = value
450
+
451
+            key = CleanText('//div[@class="title-dpe clearfix"]')(self)
452
+            value = CleanText('//div[@class="energy-consumption"]')(self)
453
+            if value and key:
454
+                details[key] = value
455
+            return details

+ 101
- 0
modules/explorimmo/test.py View File

@@ -0,0 +1,101 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from weboob.capabilities.housing import Query, ADVERT_TYPES, POSTS_TYPES
21
+from weboob.tools.capabilities.housing.housing_test import HousingTest
22
+from weboob.tools.test import BackendTest
23
+
24
+
25
+class ExplorimmoTest(BackendTest, HousingTest):
26
+    MODULE = 'explorimmo'
27
+
28
+    FIELDS_ALL_HOUSINGS_LIST = [
29
+        "id", "type", "advert_type", "house_type", "title", "location",
30
+        "utilities", "text", "area", "url"
31
+    ]
32
+    FIELDS_ANY_HOUSINGS_LIST = [
33
+        "photos", "cost", "currency"
34
+    ]
35
+    FIELDS_ALL_SINGLE_HOUSING = [
36
+        "id", "url", "type", "advert_type", "house_type", "title", "area",
37
+        "cost", "currency", "utilities", "date", "location", "text", "rooms",
38
+        "details"
39
+    ]
40
+    FIELDS_ANY_SINGLE_HOUSING = [
41
+        "bedrooms",
42
+        "photos",
43
+        "DPE",
44
+        "GES",
45
+        "phone"
46
+    ]
47
+
48
+    def test_explorimmo_rent(self):
49
+        query = Query()
50
+        query.area_min = 20
51
+        query.cost_max = 1500
52
+        query.type = POSTS_TYPES.RENT
53
+        query.cities = []
54
+        for city in self.backend.search_city('paris'):
55
+            city.backend = self.backend.name
56
+            query.cities.append(city)
57
+        self.check_against_query(query)
58
+
59
+    def test_explorimmo_sale(self):
60
+        query = Query()
61
+        query.area_min = 20
62
+        query.type = POSTS_TYPES.SALE
63
+        query.cities = []
64
+        for city in self.backend.search_city('paris'):
65
+            city.backend = self.backend.name
66
+            query.cities.append(city)
67
+        self.check_against_query(query)
68
+
69
+    def test_explorimmo_furnished_rent(self):
70
+        query = Query()
71
+        query.area_min = 20
72
+        query.cost_max = 1500
73
+        query.type = POSTS_TYPES.FURNISHED_RENT
74
+        query.cities = []
75
+        for city in self.backend.search_city('paris'):
76
+            city.backend = self.backend.name
77
+            query.cities.append(city)
78
+        self.check_against_query(query)
79
+
80
+    def test_explorimmo_viager(self):
81
+        query = Query()
82
+        query.type = POSTS_TYPES.VIAGER
83
+        query.cities = []
84
+        for city in self.backend.search_city('85'):
85
+            city.backend = self.backend.name
86
+            query.cities.append(city)
87
+        self.check_against_query(query)
88
+
89
+    def test_explorimmo_personal(self):
90
+        query = Query()
91
+        query.area_min = 20
92
+        query.cost_max = 900
93
+        query.type = POSTS_TYPES.RENT
94
+        query.advert_types = [ADVERT_TYPES.PERSONAL]
95
+        query.cities = []
96
+        for city in self.backend.search_city('paris'):
97
+            city.backend = self.backend.name
98
+            query.cities.append(city)
99
+
100
+        results = list(self.backend.search_housings(query))
101
+        self.assertEqual(len(results), 0)

+ 26
- 0
modules/foncia/__init__.py View File

@@ -0,0 +1,26 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2017      Phyks (Lucas Verney)
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from __future__ import unicode_literals
21
+
22
+
23
+from .module import FonciaModule
24
+
25
+
26
+__all__ = ['FonciaModule']

+ 61
- 0
modules/foncia/browser.py View File

@@ -0,0 +1,61 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2017      Phyks (Lucas Verney)
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from __future__ import unicode_literals
21
+
22
+
23
+from weboob.browser import PagesBrowser, URL
24
+
25
+from .constants import QUERY_TYPES
26
+from .pages import CitiesPage, HousingPage, SearchPage, SearchResultsPage
27
+
28
+
29
+class FonciaBrowser(PagesBrowser):
30
+    BASEURL = 'https://fr.foncia.com'
31
+
32
+    cities = URL(r'/recherche/autocomplete\?term=(?P<term>.+)', CitiesPage)
33
+    housing = URL(r'/(?P<type>[^/]+)/.*\d+.htm', HousingPage)
34
+    search_results = URL(r'/(?P<type>[^/]+)/.*', SearchResultsPage)
35
+    search = URL(r'/(?P<type>.+)', SearchPage)
36
+
37
+    def get_cities(self, pattern):
38
+        """
39
+        Get cities matching a given pattern.
40
+        """
41
+        return self.cities.open(term=pattern).iter_cities()
42
+
43
+    def search_housings(self, query, cities):
44
+        """
45
+        Search for housings matching given query.
46
+        """
47
+        try:
48
+            query_type = QUERY_TYPES[query.type]
49
+        except KeyError:
50
+            return []
51
+
52
+        self.search.go(type=query_type).do_search(query, cities)
53
+        return self.page.iter_housings(query_type=query.type)
54
+
55
+    def get_housing(self, housing):
56
+        """
57
+        Get specific housing.
58
+        """
59
+        query_type, housing = housing.split(':')
60
+        self.search.go(type=query_type).find_housing(query_type, housing)
61
+        return self.page.get_housing()

+ 24
- 0
modules/foncia/constants.py View File

@@ -0,0 +1,24 @@
1
+from weboob.capabilities.housing import POSTS_TYPES, HOUSE_TYPES
2
+
3
+QUERY_TYPES = {
4
+    POSTS_TYPES.RENT: 'location',
5
+    POSTS_TYPES.SALE: 'achat',
6
+    POSTS_TYPES.FURNISHED_RENT: 'location'
7
+}
8
+
9
+QUERY_HOUSE_TYPES = {
10
+    HOUSE_TYPES.APART: ['appartement', 'appartement-meuble'],
11
+    HOUSE_TYPES.HOUSE: ['maison'],
12
+    HOUSE_TYPES.PARKING: ['parking'],
13
+    HOUSE_TYPES.LAND: ['terrain'],
14
+    HOUSE_TYPES.OTHER: ['chambre', 'programme-neuf',
15
+                        'local-commercial', 'immeuble']
16
+}
17
+
18
+AVAILABLE_TYPES = {
19
+    POSTS_TYPES.RENT: ['appartement', 'maison', 'parking', 'chambre',
20
+                       'local-commercial'],
21
+    POSTS_TYPES.SALE: ['appartement', 'maison', 'parking', 'local-commercial',
22
+                       'terrain', 'immeuble', 'programme-neuf'],
23
+    POSTS_TYPES.FURNISHED_RENT: ['appartement-meuble']
24
+}

BIN
modules/foncia/favicon.png View File


+ 74
- 0
modules/foncia/module.py View File

@@ -0,0 +1,74 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2017      Phyks (Lucas Verney)
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from __future__ import unicode_literals
21
+
22
+
23
+from weboob.tools.backend import Module
24
+from weboob.capabilities.housing import CapHousing, Housing, ADVERT_TYPES, HousingPhoto
25
+
26
+from .browser import FonciaBrowser
27
+
28
+
29
+__all__ = ['FonciaModule']
30
+
31
+
32
+class FonciaModule(Module, CapHousing):
33
+    NAME = 'foncia'
34
+    DESCRIPTION = u'Foncia housing website.'
35
+    MAINTAINER = u'Phyks (Lucas Verney)'
36
+    EMAIL = 'phyks@phyks.me'
37
+    LICENSE = 'AGPLv3+'
38
+    VERSION = '2.1'
39
+
40
+    BROWSER = FonciaBrowser
41
+
42
+    def get_housing(self, housing):
43
+        return self.browser.get_housing(housing)
44
+
45
+    def search_city(self, pattern):
46
+        return self.browser.get_cities(pattern)
47
+
48
+    def search_housings(self, query):
49
+        if (
50
+                len(query.advert_types) == 1 and
51
+                query.advert_types[0] == ADVERT_TYPES.PERSONAL
52
+        ):
53
+            # Foncia is pro only
54
+            return list()
55
+
56
+        cities = ','.join(
57
+            ['%s' % c.name for c in query.cities if c.backend == self.name]
58
+        )
59
+        if len(cities) == 0:
60
+            return []
61
+
62
+        return self.browser.search_housings(query, cities)
63
+
64
+    def fill_housing(self, housing, fields):
65
+        if len(fields) > 0:
66
+            self.browser.get_housing(housing)
67
+        return housing
68
+
69
+    def fill_photo(self, photo, fields):
70
+        if 'data' in fields and photo.url and not photo.data:
71
+            photo.data = self.browser.open(photo.url).content
72
+        return photo
73
+
74
+    OBJECTS = {Housing: fill_housing, HousingPhoto: fill_photo}

+ 359
- 0
modules/foncia/pages.py View File

@@ -0,0 +1,359 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2017      Phyks (Lucas Verney)
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from __future__ import unicode_literals
21
+
22
+import datetime
23
+
24
+from weboob.browser.pages import JsonPage, HTMLPage, pagination
25
+from weboob.browser.filters.standard import (
26
+    CleanDecimal, CleanText, Currency, Date, Env, Format, Regexp, RegexpError
27
+)
28
+from weboob.browser.filters.html import AbsoluteLink, Attr, Link, XPathNotFound
29
+from weboob.browser.elements import ItemElement, ListElement, method
30
+from weboob.capabilities.base import NotAvailable, NotLoaded
31
+from weboob.capabilities.housing import (
32
+    City, Housing, HousingPhoto,
33
+    UTILITIES, ENERGY_CLASS, POSTS_TYPES, ADVERT_TYPES
34
+)
35
+from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
36
+
37
+from .constants import AVAILABLE_TYPES, QUERY_TYPES, QUERY_HOUSE_TYPES
38
+
39
+
40
+class CitiesPage(JsonPage):
41
+    def iter_cities(self):
42
+        cities_list = self.doc
43
+        if isinstance(self.doc, dict):
44
+            cities_list = self.doc.values()
45
+
46
+        for city in cities_list:
47
+            city_obj = City()
48
+            city_obj.id = city
49
+            city_obj.name = city
50
+            yield city_obj
51
+
52
+
53
+class HousingPage(HTMLPage):
54
+    @method
55
+    class get_housing(ItemElement):
56
+        klass = Housing
57
+
58
+        obj_id = Format(
59
+            '%s:%s',
60
+            Env('type'),
61
+            Attr('//div[boolean(@data-property-reference)]', 'data-property-reference')
62
+        )
63
+        obj_advert_type = ADVERT_TYPES.PROFESSIONAL
64
+
65
+        def obj_type(self):
66
+            type = Env('type')(self)
67
+            if type == 'location':
68
+                if 'appartement-meuble' in self.page.url:
69
+                    return POSTS_TYPES.FURNISHED_RENT
70
+                else:
71
+                    return POSTS_TYPES.RENT
72
+            elif type == 'achat':
73
+                return POSTS_TYPES.SALE
74
+            else:
75
+                return NotAvailable
76
+
77
+        def obj_url(self):
78
+            return self.page.url
79
+
80
+        def obj_house_type(self):
81
+            url = self.obj_url()
82
+            for house_type, types in QUERY_HOUSE_TYPES.items():
83
+                for type in types:
84
+                    if ('/%s/' % type) in url:
85
+                        return house_type
86
+            return NotAvailable
87
+
88
+        obj_title = CleanText('//h1[has-class("OfferTop-title")]')
89
+        obj_area = CleanDecimal(
90
+            Regexp(
91
+                CleanText(
92
+                    '//div[has-class("MiniData")]//p[has-class("MiniData-item")][1]'
93
+                ),
94
+                r'(\d*\.*\d*) .*',
95
+                default=NotAvailable
96
+            ),
97
+            default=NotAvailable
98
+        )
99
+        obj_cost = CleanDecimal(
100
+            '//span[has-class("OfferTop-price")]',
101
+            default=NotAvailable
102
+        )
103
+        obj_price_per_meter = PricePerMeterFilter()
104
+        obj_currency = Currency(
105
+            '//span[has-class("OfferTop-price")]'
106
+        )
107
+        obj_location = Format(
108
+            '%s - %s',
109
+            CleanText('//p[@data-behat="adresseBien"]'),
110
+            CleanText('//p[has-class("OfferTop-loc")]')
111
+        )
112
+        obj_text = CleanText('//div[has-class("OfferDetails-content")]/p[1]')
113
+        obj_phone = Regexp(
114
+            Link(
115
+                '//a[has-class("OfferContact-btn--tel")]'
116
+            ),
117
+            r'tel:(.*)'
118
+        )
119
+
120
+        def obj_photos(self):
121
+            photos = []
122
+            for photo in self.xpath('//div[has-class("OfferSlider")]//img'):
123
+                photo_url = Attr('.', 'src')(photo)
124
+                photo_url = photo_url.replace('640/480', '800/600')
125
+                photos.append(HousingPhoto(photo_url))
126
+            return photos
127
+
128
+        obj_date = datetime.date.today()
129
+
130
+        def obj_utilities(self):
131
+            price = CleanText(
132
+                '//p[has-class("OfferTop-price")]'
133
+            )(self)
134
+            if "charges comprises" in price.lower():
135
+                return UTILITIES.INCLUDED
136
+            else:
137
+                return UTILITIES.EXCLUDED
138
+
139
+        obj_rooms = CleanDecimal(
140
+            '//div[has-class("MiniData")]//p[has-class("MiniData-item")][2]',
141
+            default=NotAvailable
142
+        )
143
+        obj_bedrooms = CleanDecimal(
144
+            '//div[has-class("MiniData")]//p[has-class("MiniData-item")][3]',
145
+            default=NotAvailable
146
+        )
147
+
148
+        def obj_DPE(self):
149
+            try:
150
+                electric_consumption = CleanDecimal(Regexp(
151
+                    Attr('//div[has-class("OfferDetails-content")]//img', 'src'),
152
+                    r'https://dpe.foncia.net\/(\d+)\/.*'
153
+                ))(self)
154
+            except (RegexpError, XPathNotFound):
155
+                electric_consumption = None
156
+
157
+            DPE = ""
158
+            if electric_consumption is not None:
159
+                if electric_consumption <= 50:
160
+                    DPE = "A"
161
+                elif 50 < electric_consumption <= 90:
162
+                    DPE = "B"
163
+                elif 90 < electric_consumption <= 150:
164
+                    DPE = "C"
165
+                elif 150 < electric_consumption <= 230:
166
+                    DPE = "D"
167
+                elif 230 < electric_consumption <= 330:
168
+                    DPE = "E"
169
+                elif 330 < electric_consumption <= 450:
170
+                    DPE = "F"
171
+                else:
172
+                    DPE = "G"
173
+                return getattr(ENERGY_CLASS, DPE, NotAvailable)
174
+            return NotAvailable
175
+
176
+        def obj_details(self):
177
+            details = {}
178
+
179
+            dispo = Date(
180
+                Regexp(
181
+                    CleanText('//p[has-class("OfferTop-dispo")]'),
182
+                    r'.* (\d\d\/\d\d\/\d\d\d\d)',
183
+                    default=datetime.date.today().isoformat()
184
+                )
185
+            )(self)
186
+            if dispo is not None:
187
+                details["dispo"] = dispo
188
+
189
+            priceMentions = CleanText(
190
+                '//p[has-class("OfferTop-mentions")]',
191
+                default=None
192
+            )(self)
193
+            if priceMentions is not None:
194
+                details["priceMentions"] = priceMentions
195
+
196
+            agency = CleanText(
197
+                '//p[has-class("OfferContact-address")]',
198
+                default=None
199
+            )(self)
200
+            if agency is not None:
201
+                details["agency"] = agency
202
+
203
+            for item in self.xpath('//div[has-class("OfferDetails-columnize")]/div'):
204
+                category = CleanText(
205
+                    './h3[has-class("OfferDetails-title--2")]',
206
+                    default=None
207
+                )(item)
208
+                if not category:
209
+                    continue
210
+
211
+                details[category] = {}
212
+
213
+                for detail_item in item.xpath('.//ul[has-class("List--data")]/li'):
214
+                    detail_title = CleanText('.//span[has-class("List-data")]')(detail_item)
215
+                    detail_value = CleanText('.//*[has-class("List-value")]')(detail_item)
216
+                    details[category][detail_title] = detail_value
217
+
218
+                for detail_item in item.xpath('.//ul[has-class("List--bullet")]/li'):
219
+                    detail_title = CleanText('.')(detail_item)
220
+                    details[category][detail_title] = True
221
+
222
+            try:
223
+                electric_consumption = CleanDecimal(Regexp(
224
+                    Attr('//div[has-class("OfferDetails-content")]//img', 'src'),
225
+                    r'https://dpe.foncia.net\/(\d+)\/.*'
226
+                ))(self)
227
+                details["electric_consumption"] = (
228
+                    '{} kWhEP/m².an'.format(electric_consumption)
229
+                )
230
+            except (RegexpError, XPathNotFound):
231
+                pass
232
+
233
+            return details
234
+
235
+
236
+class SearchPage(HTMLPage):
237
+    def do_search(self, query, cities):
238
+        form = self.get_form('//form[@name="searchForm"]')
239
+
240
+        form['searchForm[type]'] = QUERY_TYPES.get(query.type, None)
241
+        form['searchForm[localisation]'] = cities
242
+        form['searchForm[type_bien][]'] = []
243
+        for house_type in query.house_types:
244
+            try:
245
+                form['searchForm[type_bien][]'].extend(
246
+                    QUERY_HOUSE_TYPES[house_type]
247
+                )
248
+            except KeyError:
249
+                pass
250
+        form['searchForm[type_bien][]'] = [
251
+            x for x in form['searchForm[type_bien][]']
252
+            if x in AVAILABLE_TYPES.get(query.type, [])
253
+        ]
254
+        if query.area_min:
255
+            form['searchForm[surface_min]'] = query.area_min
256
+        if query.area_max:
257
+            form['searchForm[surface_max]'] = query.area_max
258
+        if query.cost_min:
259
+            form['searchForm[prix_min]'] = query.cost_min
260
+        if query.cost_max:
261
+            form['searchForm[prix_max]'] = query.cost_max
262
+        if query.nb_rooms:
263
+            form['searchForm[pieces]'] = [i for i in range(1, query.nb_rooms + 1)]
264
+        form.submit()
265
+
266
+    def find_housing(self, query_type, housing):
267
+        form = self.get_form('//form[@name="searchForm"]')
268
+        form['searchForm[type]'] = query_type
269
+        form['searchForm[reference]'] = housing
270
+        form.submit()
271
+
272
+
273
+class SearchResultsPage(HTMLPage):
274
+    @pagination
275
+    @method
276
+    class iter_housings(ListElement):
277
+        item_xpath = '//article[has-class("TeaserOffer")]'
278
+
279
+        next_page = Link('//div[has-class("Pagination--more")]/a[contains(text(), "Suivant")]')
280
+
281
+        class item(ItemElement):
282
+            klass = Housing
283
+
284
+            obj_id = Format(
285
+                '%s:%s',
286
+                Env('type'),
287
+                Attr('.//span[boolean(@data-reference)]', 'data-reference')
288
+            )
289
+            obj_url = AbsoluteLink('.//h3[has-class("TeaserOffer-title")]/a')
290
+            obj_type = Env('query_type')
291
+            obj_advert_type = ADVERT_TYPES.PROFESSIONAL
292
+
293
+            def obj_house_type(self):
294
+                url = self.obj_url(self)
295
+                for house_type, types in QUERY_HOUSE_TYPES.items():
296
+                    for type in types:
297
+                        if ('/%s/' % type) in url:
298
+                            return house_type
299
+                return NotLoaded
300
+
301
+            obj_url = AbsoluteLink('.//h3[has-class("TeaserOffer-title")]/a')
302
+            obj_title = CleanText('.//h3[has-class("TeaserOffer-title")]')
303
+            obj_area = CleanDecimal(
304
+                Regexp(
305
+                    CleanText(
306
+                        './/div[has-class("MiniData")]//p[@data-behat="surfaceDesBiens"]'
307
+                    ),
308
+                    r'(\d*\.*\d*) .*',
309
+                    default=NotAvailable
310
+                ),
311
+                default=NotAvailable
312
+            )
313
+            obj_cost = CleanDecimal(
314
+                './/strong[has-class("TeaserOffer-price-num")]',
315
+                default=NotAvailable
316
+            )
317
+            obj_price_per_meter = PricePerMeterFilter()
318
+            obj_currency = Currency(
319
+                './/strong[has-class("TeaserOffer-price-num")]'
320
+            )
321
+            obj_location = CleanText('.//p[has-class("TeaserOffer-loc")]')
322
+            obj_text = CleanText('.//p[has-class("TeaserOffer-description")]')
323
+
324
+            def obj_photos(self):
325
+                url = CleanText(Attr('.//a[has-class("TeaserOffer-ill")]/img', 'src'))(self)
326
+                # If the used photo is a default no photo, the src is on the same domain.
327
+                if url[0] == '/':
328
+                    return []
329
+                else:
330
+                    return [HousingPhoto(url)]
331
+
332
+            obj_date = datetime.date.today()
333
+
334
+            def obj_utilities(self):
335
+                price = CleanText(
336
+                    './/strong[has-class("TeaserOffer-price-num")]'
337
+                )(self)
338
+                if "charges comprises" in price.lower():
339
+                    return UTILITIES.INCLUDED
340
+                else:
341
+                    return UTILITIES.EXCLUDED
342
+
343
+            obj_rooms = CleanDecimal(
344
+                './/div[has-class("MiniData")]//p[@data-behat="nbPiecesDesBiens"]',
345
+                default=NotLoaded
346
+            )
347
+            obj_bedrooms = CleanDecimal(
348
+                './/div[has-class("MiniData")]//p[@data-behat="nbChambresDesBiens"]',
349
+                default=NotLoaded
350
+            )
351
+
352
+            def obj_details(self):
353
+                return {
354
+                    "dispo": Date(
355
+                        Attr('.//span[boolean(@data-dispo)]', 'data-dispo',
356
+                             default=datetime.date.today().isoformat())
357
+                    )(self),
358
+                    "priceMentions": CleanText('.//span[has-class("TeaserOffer-price-mentions")]')(self)
359
+                }

+ 95
- 0
modules/foncia/test.py View File

@@ -0,0 +1,95 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2017      Phyks (Lucas Verney)
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from __future__ import unicode_literals
21
+
22
+from weboob.capabilities.housing import (
23
+    Query, POSTS_TYPES, ADVERT_TYPES
24
+)
25
+from weboob.tools.capabilities.housing.housing_test import HousingTest
26
+from weboob.tools.test import BackendTest
27
+
28
+
29
+class FonciaTest(BackendTest, HousingTest):
30
+    MODULE = 'foncia'
31
+
32
+    FIELDS_ALL_HOUSINGS_LIST = [
33
+        "id", "type", "advert_type", "house_type", "url", "title", "area",
34
+        "cost", "currency", "date", "location", "text", "details"
35
+    ]
36
+    FIELDS_ANY_HOUSINGS_LIST = [
37
+        "photos",
38
+        "rooms"
39
+    ]
40
+    FIELDS_ALL_SINGLE_HOUSING = [
41
+        "id", "url", "type", "advert_type", "house_type", "title", "area",
42
+        "cost", "currency", "utilities", "date", "location", "text", "phone",
43
+        "DPE", "details"
44
+    ]
45
+    FIELDS_ANY_SINGLE_HOUSING = [
46
+        "bedrooms",
47
+        "photos",
48
+        "rooms"
49
+    ]
50
+
51
+    def test_foncia_rent(self):
52
+        query = Query()
53
+        query.area_min = 20
54
+        query.cost_max = 1500
55
+        query.type = POSTS_TYPES.RENT
56
+        query.cities = []
57
+        for city in self.backend.search_city('paris'):
58
+            city.backend = self.backend.name
59
+            query.cities.append(city)
60
+        self.check_against_query(query)
61
+
62
+    def test_foncia_sale(self):
63
+        query = Query()
64
+        query.area_min = 20
65
+        query.type = POSTS_TYPES.SALE
66
+        query.cities = []
67
+        for city in self.backend.search_city('paris'):
68
+            city.backend = self.backend.name
69
+            query.cities.append(city)
70
+        self.check_against_query(query)
71
+
72
+    def test_foncia_furnished_rent(self):
73
+        query = Query()
74
+        query.area_min = 20
75
+        query.cost_max = 1500
76
+        query.type = POSTS_TYPES.FURNISHED_RENT
77
+        query.cities = []
78
+        for city in self.backend.search_city('paris'):
79
+            city.backend = self.backend.name
80
+            query.cities.append(city)
81
+        self.check_against_query(query)
82
+
83
+    def test_foncia_personal(self):
84
+        query = Query()
85
+        query.area_min = 20
86
+        query.cost_max = 900
87
+        query.type = POSTS_TYPES.RENT
88
+        query.advert_types = [ADVERT_TYPES.PERSONAL]
89
+        query.cities = []
90
+        for city in self.backend.search_city('paris'):
91
+            city.backend = self.backend.name
92
+            query.cities.append(city)
93
+
94
+        results = list(self.backend.search_housings(query))
95
+        self.assertEqual(len(results), 0)

+ 24
- 0
modules/leboncoin/__init__.py View File

@@ -0,0 +1,24 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+
21
+from .module import LeboncoinModule
22
+
23
+
24
+__all__ = ['LeboncoinModule']

+ 145
- 0
modules/leboncoin/browser.py View File

@@ -0,0 +1,145 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from weboob.tools.json import json
21
+
22
+from weboob.browser import PagesBrowser, URL
23
+from weboob.capabilities.housing import (TypeNotSupported, POSTS_TYPES,
24
+                                         HOUSE_TYPES, ADVERT_TYPES)
25
+from .pages import CityListPage, HousingListPage, HousingPage, PhonePage, HomePage
26
+
27
+
28
+class LeboncoinBrowser(PagesBrowser):
29
+    BASEURL = 'https://www.leboncoin.fr/'
30
+    city = URL('ajax/location_list.html\?city=(?P<city>.*)&zipcode=(?P<zip>.*)', CityListPage)
31
+    housing = URL('ventes_immobilieres/(?P<_id>.*).htm', HousingPage)
32
+
33
+    home = URL('annonces/offres', HomePage)
34
+    api = URL('https://api.leboncoin.fr/finder/search', HousingListPage)
35
+    phone = URL('https://api.leboncoin.fr/api/utils/phonenumber.json', PhonePage)
36
+
37
+    TYPES = {POSTS_TYPES.RENT: '10',
38
+             POSTS_TYPES.FURNISHED_RENT: '10',
39
+             POSTS_TYPES.SALE: '9',
40
+             POSTS_TYPES.SHARING: '11', }
41
+
42
+    RET = {HOUSE_TYPES.HOUSE: '1',
43
+           HOUSE_TYPES.APART: '2',
44
+           HOUSE_TYPES.LAND: '3',
45
+           HOUSE_TYPES.PARKING: '4',
46
+           HOUSE_TYPES.OTHER: '5'}
47
+
48
+    def __init__(self, *args, **kwargs):
49
+        super(LeboncoinBrowser, self).__init__(*args, **kwargs)
50
+
51
+    def get_cities(self, pattern):
52
+        city = ''
53
+        zip_code = ''
54
+        if pattern.isdigit():
55
+            zip_code = pattern
56
+        else:
57
+            city = pattern.replace(" ", "_")
58
+
59
+        return self.city.go(city=city, zip=zip_code).get_cities()
60
+
61
+    def search_housings(self, query, module_name):
62
+
63
+        if query.type not in self.TYPES.keys():
64
+            return TypeNotSupported()
65
+
66
+        data = {}
67
+        data['filters'] = {}
68
+        data['filters']['category'] = {}
69
+        data['filters']['category']['id'] = self.TYPES.get(query.type)
70
+        data['filters']['enums'] = {}
71
+        data['filters']['enums']['ad_type'] = ['offer']
72
+
73
+        data['filters']['enums']['real_estate_type'] = []
74
+        for t in query.house_types:
75
+            t = self.RET.get(t)
76
+            if t:
77
+                data['filters']['enums']['real_estate_type'].append(t)
78
+
79
+        if query.type == POSTS_TYPES.FURNISHED_RENT:
80
+            data['filters']['enums']['furnished'] = ['1']
81
+        elif query.type == POSTS_TYPES.RENT:
82
+            data['filters']['enums']['furnished'] = ['2']
83
+
84
+        data['filters']['keywords'] = {}
85
+        data['filters']['ranges'] = {}
86
+
87
+        if query.cost_max or query.cost_min:
88
+            data['filters']['ranges']['price'] = {}
89
+
90
+            if query.cost_max:
91
+                data['filters']['ranges']['price']['max'] = query.cost_max
92
+
93
+                if query.cost_min:
94
+                    data['filters']['ranges']['price']['min'] = query.cost_min
95
+
96
+        if query.area_max or query.area_min:
97
+            data['filters']['ranges']['square'] = {}
98
+            if query.area_max:
99
+                data['filters']['ranges']['square']['max'] = query.area_max
100
+
101
+            if query.area_min:
102
+                data['filters']['ranges']['square']['min'] = query.area_min
103
+
104
+        if query.nb_rooms:
105
+            data['filters']['ranges']['rooms'] = {}
106
+            data['filters']['ranges']['rooms']['min'] = query.nb_rooms
107
+
108
+        data['filters']['location'] = {}
109
+        data['filters']['location']['city_zipcodes'] = []
110
+
111
+        for c in query.cities:
112
+            if c.backend == module_name:
113
+                _c = c.id.split(' ')
114
+                __c = {}
115
+                __c['city'] = _c[0]
116
+                __c['zipcode'] = _c[1]
117
+                __c['label'] = c.name
118
+
119
+                data['filters']['location']['city_zipcodes'].append(__c)
120
+
121
+        if len(query.advert_types) == 1:
122
+            if query.advert_types[0] == ADVERT_TYPES.PERSONAL:
123
+                data['owner_type'] = 'private'
124
+            elif query.advert_types[0] == ADVERT_TYPES.PROFESSIONAL:
125
+                data['owner_type'] = 'pro'
126
+        else:
127
+            data['owner_type'] = 'all'
128
+
129
+        data['limit'] = 100
130
+        data['limit_alu'] = 3
131
+        data['offset'] = 0
132
+
133
+        self.session.headers.update({"api_key": self.home.go().get_api_key()})
134
+        return self.api.go(data=json.dumps(data)).get_housing_list(query_type=query.type, data=data)
135
+
136
+    def get_housing(self, _id, obj=None):
137
+        return self.housing.go(_id=_id).get_housing(obj=obj)
138
+
139
+    def get_phone(self, _id):
140
+        api_key = self.housing.stay_or_go(_id=_id).get_api_key()
141
+        data = {'list_id': _id,
142
+                'app_id': 'leboncoin_web_utils',
143
+                'key': api_key,
144
+                'text': 1, }
145
+        return self.phone.go(data=data).get_phone()

BIN
modules/leboncoin/favicon.png View File


+ 66
- 0
modules/leboncoin/module.py View File

@@ -0,0 +1,66 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+
21
+from weboob.tools.backend import Module
22
+from weboob.capabilities.housing import (CapHousing, Housing, HousingPhoto)
23
+from .browser import LeboncoinBrowser
24
+
25
+
26
+__all__ = ['LeboncoinModule']
27
+
28
+
29
+class LeboncoinModule(Module, CapHousing):
30
+    NAME = 'leboncoin'
31
+    DESCRIPTION = u'search house on leboncoin website'
32
+    MAINTAINER = u'Bezleputh'
33
+    EMAIL = 'carton_ben@yahoo.fr'
34
+    LICENSE = 'AGPLv3+'
35
+    VERSION = '2.1'
36
+
37
+    BROWSER = LeboncoinBrowser
38
+
39
+    def create_default_browser(self):
40
+        return self.create_browser()
41
+
42
+    def get_housing(self, _id):
43
+        return self.browser.get_housing(_id)
44
+
45
+    def fill_housing(self, housing, fields):
46
+        if 'phone' in fields:
47
+            housing.phone = self.browser.get_phone(housing.id)
48
+            fields.remove('phone')
49
+
50
+        if len(fields) > 0:
51
+            self.browser.get_housing(housing.id, housing)
52
+
53
+        return housing
54
+
55
+    def fill_photo(self, photo, fields):
56
+        if 'data' in fields and photo.url and not photo.data:
57
+            photo.data = self.browser.open(photo.url).content
58
+        return photo
59
+
60
+    def search_city(self, pattern):
61
+        return self.browser.get_cities(pattern)
62
+
63
+    def search_housings(self, query):
64
+        return self.browser.search_housings(query, self.name)
65
+
66
+    OBJECTS = {Housing: fill_housing, HousingPhoto: fill_photo}

+ 301
- 0
modules/leboncoin/pages.py View File

@@ -0,0 +1,301 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+from __future__ import unicode_literals
20
+
21
+import requests
22
+
23
+from weboob.browser.pages import HTMLPage, JsonPage, pagination
24
+from weboob.browser.elements import ItemElement, ListElement, method, DictElement
25
+from weboob.capabilities.base import Currency as BaseCurrency
26
+from weboob.browser.filters.standard import (CleanText, CleanDecimal, _Filter,
27
+                                             Env, DateTime, Format)
28
+from weboob.browser.filters.json import Dict
29
+from weboob.capabilities.housing import (City, Housing, HousingPhoto,
30
+                                         UTILITIES, ENERGY_CLASS, POSTS_TYPES,
31
+                                         ADVERT_TYPES, HOUSE_TYPES)
32
+from weboob.capabilities.base import NotAvailable
33
+from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
34
+
35
+from decimal import Decimal
36
+from lxml import etree
37
+import json
38
+
39
+
40
+class PopDetail(_Filter):
41
+    def __init__(self, name, default=NotAvailable):
42
+        super(PopDetail, self).__init__(default)
43
+        self.name = name
44
+
45
+    def __call__(self, item):
46
+        return item.env['details'].pop(self.name, self.default)
47
+
48
+
49
+class CityListPage(HTMLPage):
50
+
51
+    def build_doc(self, content):
52
+        content = super(CityListPage, self).build_doc(content)
53
+        if content.getroot() is not None:
54
+            return content
55
+        return etree.Element("html")
56
+
57
+    @method
58
+    class get_cities(ListElement):
59
+        item_xpath = '//li'
60
+
61
+        class item(ItemElement):
62
+            klass = City
63
+
64
+            obj_id = Format('%s %s',
65
+                            CleanText('./span[has-class("city")]'),
66
+                            CleanText('./span[@class="zipcode"]'))
67
+
68
+            obj_name = Format('%s %s',
69
+                              CleanText('./span[has-class("city")]'),
70
+                              CleanText('./span[@class="zipcode"]'))
71
+
72
+
73
+class HomePage(HTMLPage):
74
+    def __init__(self, *args, **kwargs):
75
+        HTMLPage.__init__(self, *args, **kwargs)
76
+
77
+        add_content = CleanText('(//body/script)[4]', replace=[('window.FLUX_STATE = ', '')])(self.doc) or '{}'
78
+        api_content = CleanText('(//body/script[@id="__NEXT_DATA__"])')(self.doc)
79
+
80
+        self.htmldoc = self.doc
81
+        self.api_content = json.loads(api_content)
82
+        self.doc = json.loads(add_content)
83
+
84
+    def get_api_key(self):
85
+        return Dict('runtimeConfig/API/KEY')(self.api_content)
86
+
87
+
88
+class HousingListPage(JsonPage):
89
+
90
+    def __init__(self, *args, **kwargs):
91
+        JsonPage.__init__(self, *args, **kwargs)
92
+        if 'ads' not in self.doc:
93
+            self.doc['ads'] = []
94
+
95
+    @pagination
96
+    @method
97
+    class get_housing_list(DictElement):
98
+        item_xpath = 'ads'
99
+
100
+        def next_page(self):
101
+            data = Env('data')(self)
102
+            if data['offset'] > self.page.doc['total_all']:
103
+                return
104
+
105
+            data['offset'] = data['offset'] + data['limit']
106
+            return requests.Request("POST", self.page.url, data=json.dumps(data))
107
+
108
+        class item(ItemElement):
109
+            klass = Housing
110
+
111
+            def parse(self, el):
112
+                self.env['details'] = {obj['key']: obj['value_label'] for obj in self.el['attributes']}
113
+
114
+            obj_id = Dict('list_id')
115
+            obj_url = Dict('url')
116
+            obj_type = Env('query_type')
117
+
118
+            obj_area = CleanDecimal(PopDetail('square',
119
+                                              default=0),
120
+                                    default=NotAvailable)
121
+            obj_rooms = CleanDecimal(PopDetail('rooms',
122
+                                               default=0),
123
+                                     default=NotAvailable)
124
+
125
+            def obj_GES(self):
126
+                ges = CleanText(PopDetail('ges', default='|'))(self)
127
+                return getattr(ENERGY_CLASS, ges[0], NotAvailable)
128
+
129
+            def obj_DPE(self):
130
+                dpe = CleanText(PopDetail('energy_rate', default='|'))(self)
131
+                return getattr(ENERGY_CLASS, dpe[0], NotAvailable)
132
+
133
+            def obj_house_type(self):
134
+                value = CleanText(PopDetail('real_estate_type'), default=' ')(self).lower()
135
+                if value == 'parking':
136
+                    return HOUSE_TYPES.PARKING
137
+                elif value == 'appartement':
138
+                    return HOUSE_TYPES.APART
139
+                elif value == 'maison':
140
+                    return HOUSE_TYPES.HOUSE
141
+                elif value == 'terrain':
142
+                    return HOUSE_TYPES.LAND
143
+                else:
144
+                    return HOUSE_TYPES.OTHER
145
+
146
+            def obj_utilities(self):
147
+                value = CleanText(PopDetail('charges_included',
148
+                                            default='Non'),
149
+                                  default=NotAvailable)(self)
150
+                if value == "Oui":
151
+                    return UTILITIES.INCLUDED
152
+                else:
153
+                    return UTILITIES.EXCLUDED
154
+
155
+            def obj_advert_type(self):
156
+                line_pro = Dict('owner/type')(self)
157
+                if line_pro == u'pro':
158
+                    return ADVERT_TYPES.PROFESSIONAL
159
+                else:
160
+                    return ADVERT_TYPES.PERSONAL
161
+
162
+            obj_title = Dict('subject')
163
+            obj_cost = CleanDecimal(Dict('price/0', default=NotAvailable), default=Decimal(0))
164
+            obj_currency = BaseCurrency.get_currency(u'€')
165
+            obj_text = Dict('body')
166
+            obj_location = Dict('location/city_label')
167
+            obj_date = DateTime(Dict('first_publication_date'))
168
+
169
+            def obj_photos(self):
170
+                photos = []
171
+                for img in Dict('images/urls_large', default=[])(self):
172
+                    photos.append(HousingPhoto(img))
173
+                return photos
174
+
175
+            def obj_type(self):
176
+                try:
177
+                    breadcrumb = int(Dict('category_id')(self))
178
+                except ValueError:
179
+                    breadcrumb = None
180
+
181
+                if breadcrumb == 11:
182
+                    return POSTS_TYPES.SHARING
183
+                elif breadcrumb == 10:
184
+
185
+                    isFurnished = CleanText(PopDetail('furnished', default=' '))(self)
186
+
187
+                    if isFurnished.lower() == u'meublé':
188
+                        return POSTS_TYPES.FURNISHED_RENT
189
+                    else:
190
+                        return POSTS_TYPES.RENT
191
+                else:
192
+                    return POSTS_TYPES.SALE
193
+
194
+            obj_price_per_meter = PricePerMeterFilter()
195
+            obj_details = Env('details')
196
+
197
+
198
+class HousingPage(HomePage):
199
+    def __init__(self, *args, **kwargs):
200
+        HomePage.__init__(self, *args, **kwargs)
201
+        self.doc = self.api_content["props"]["pageProps"]["ad"]
202
+
203
+    def get_api_key(self):
204
+        return Dict('runtimeConfig/API/KEY_JSON')(self.api_content)
205
+
206
+    @method
207
+    class get_housing(ItemElement):
208
+        klass = Housing
209
+
210
+        def parse(self, el):
211
+            self.env['details'] = {obj['key']: obj['value_label'] for obj in el['attributes']}
212
+
213
+        obj_id = Env('_id')
214
+
215
+        obj_area = CleanDecimal(PopDetail('square',
216
+                                          default=0),
217
+                                default=NotAvailable)
218
+        obj_rooms = CleanDecimal(PopDetail('rooms',
219
+                                           default=0),
220
+                                 default=NotAvailable)
221
+
222
+        def obj_GES(self):
223
+            ges = CleanText(PopDetail('ges', default='|'))(self)
224
+            return getattr(ENERGY_CLASS, ges[0], NotAvailable)
225
+
226
+        def obj_DPE(self):
227
+            dpe = CleanText(PopDetail('energy_rate', default='|'))(self)
228
+            return getattr(ENERGY_CLASS, dpe[0], NotAvailable)
229
+
230
+        def obj_house_type(self):
231
+            value = CleanText(PopDetail('real_estate_type'), default=' ')(self).lower()
232
+            if value == 'parking':
233
+                return HOUSE_TYPES.PARKING
234
+            elif value == 'appartement':
235
+                return HOUSE_TYPES.APART
236
+            elif value == 'maison':
237
+                return HOUSE_TYPES.HOUSE
238
+            elif value == 'terrain':
239
+                return HOUSE_TYPES.LAND
240
+            else:
241
+                return HOUSE_TYPES.OTHER
242
+
243
+        def obj_utilities(self):
244
+            value = CleanText(PopDetail('charges_included',
245
+                                        default='Non'),
246
+                              default=NotAvailable)(self)
247
+            if value == "Oui":
248
+                return UTILITIES.INCLUDED
249
+            else:
250
+                return UTILITIES.EXCLUDED
251
+
252
+        obj_title = Dict('subject')
253
+        obj_cost = CleanDecimal(Dict('price/0', default=NotAvailable), default=Decimal(0))
254
+        obj_currency = BaseCurrency.get_currency(u'€')
255
+        obj_text = Dict('body')
256
+        obj_location = Dict('location/city_label')
257
+
258
+        def obj_advert_type(self):
259
+            line_pro = Dict('owner/type')(self)
260
+            if line_pro == u'pro':
261
+                return ADVERT_TYPES.PROFESSIONAL
262
+            else:
263
+                return ADVERT_TYPES.PERSONAL
264
+
265
+        obj_date = DateTime(Dict('first_publication_date'))
266
+
267
+        def obj_photos(self):
268
+            photos = []
269
+            for img in Dict('images/urls_large', default=[])(self):
270
+                photos.append(HousingPhoto(img))
271
+            return photos
272
+
273
+        def obj_type(self):
274
+            try:
275
+                breadcrumb = int(Dict('category_id')(self))
276
+            except ValueError:
277
+                breadcrumb = None
278
+
279
+            if breadcrumb == 11:
280
+                return POSTS_TYPES.SHARING
281
+            elif breadcrumb == 10:
282
+
283
+                isFurnished = CleanText(PopDetail('furnished', default=' '))(self)
284
+
285
+                if isFurnished.lower() == u'meublé':
286
+                    return POSTS_TYPES.FURNISHED_RENT
287
+                else:
288
+                    return POSTS_TYPES.RENT
289
+            else:
290
+                return POSTS_TYPES.SALE
291
+
292
+        obj_price_per_meter = PricePerMeterFilter()
293
+        obj_url = Dict('url')
294
+        obj_details = Env('details')
295
+
296
+
297
+class PhonePage(JsonPage):
298
+    def get_phone(self):
299
+        if Dict('utils/status')(self.doc) == u'OK':
300
+            return Dict('utils/phonenumber')(self.doc)
301
+        return NotAvailable

+ 105
- 0
modules/leboncoin/test.py View File

@@ -0,0 +1,105 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from weboob.tools.test import BackendTest
21
+from weboob.tools.value import Value
22
+from weboob.capabilities.housing import Query, POSTS_TYPES, ADVERT_TYPES
23
+from weboob.tools.capabilities.housing.housing_test import HousingTest
24
+
25
+
26
+class LeboncoinTest(BackendTest, HousingTest):
27
+    MODULE = 'leboncoin'
28
+
29
+    FIELDS_ALL_HOUSINGS_LIST = [
30
+        "id", "type", "advert_type", "url", "title",
31
+        "currency", "utilities", "date", "location", "text"
32
+    ]
33
+    FIELDS_ANY_HOUSINGS_LIST = [
34
+        "area",
35
+        "cost",
36
+        "price_per_meter",
37
+        "photos"
38
+    ]
39
+    FIELDS_ALL_SINGLE_HOUSING = [
40
+        "id", "url", "type", "advert_type", "house_type", "title",
41
+        "cost", "currency", "utilities", "date", "location", "text",
42
+        "rooms", "details"
43
+    ]
44
+    FIELDS_ANY_SINGLE_HOUSING = [
45
+        "area",
46
+        "GES",
47
+        "DPE",
48
+        "photos",
49
+        # Don't test phone as leboncoin API is strongly rate-limited
50
+    ]
51
+
52
+    def setUp(self):
53
+        if not self.is_backend_configured():
54
+            self.backend.config['advert_type'] = Value(value='a')
55
+            self.backend.config['region'] = Value(value='ile_de_france')
56
+
57
+    def test_leboncoin_rent(self):
58
+        query = Query()
59
+        query.area_min = 20
60
+        query.cost_max = 1500
61
+        query.type = POSTS_TYPES.RENT
62
+        query.cities = []
63
+        for city in self.backend.search_city('paris'):
64
+            city.backend = self.backend.name
65
+            query.cities.append(city)
66
+            if len(query.cities) == 3:
67
+                break
68
+        self.check_against_query(query)
69
+
70
+    def test_leboncoin_sale(self):
71
+        query = Query()
72
+        query.area_min = 20
73
+        query.type = POSTS_TYPES.SALE
74
+        query.cities = []
75
+        for city in self.backend.search_city('paris'):
76
+            city.backend = self.backend.name
77
+            query.cities.append(city)
78
+            if len(query.cities) == 3:
79
+                break
80
+        self.check_against_query(query)
81
+
82
+    def test_leboncoin_furnished_rent(self):
83
+        query = Query()
84
+        query.area_min = 20
85
+        query.cost_max = 1500
86
+        query.type = POSTS_TYPES.FURNISHED_RENT
87
+        query.cities = []
88
+        for city in self.backend.search_city('paris'):
89
+            city.backend = self.backend.name
90
+            query.cities.append(city)
91
+            if len(query.cities) == 3:
92
+                break
93
+        self.check_against_query(query)
94
+
95
+    def test_leboncoin_professional(self):
96
+        query = Query()
97
+        query.area_min = 20
98
+        query.cost_max = 900
99
+        query.type = POSTS_TYPES.RENT
100
+        query.advert_types = [ADVERT_TYPES.PROFESSIONAL]
101
+        query.cities = []
102
+        for city in self.backend.search_city('paris'):
103
+            city.backend = self.backend.name
104
+            query.cities.append(city)
105
+        self.check_against_query(query)

+ 24
- 0
modules/logicimmo/__init__.py View File

@@ -0,0 +1,24 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+
21
+from .module import LogicimmoModule
22
+
23
+
24
+__all__ = ['LogicimmoModule']

+ 108
- 0
modules/logicimmo/browser.py View File

@@ -0,0 +1,108 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+
21
+from weboob.browser import PagesBrowser, URL
22
+from weboob.browser.profiles import Firefox
23
+from weboob.capabilities.housing import (TypeNotSupported, POSTS_TYPES,
24
+                                         HOUSE_TYPES)
25
+from .pages import CitiesPage, SearchPage, HousingPage, PhonePage
26
+
27
+
28
+class LogicimmoBrowser(PagesBrowser):
29
+    BASEURL = 'https://www.logic-immo.com/'
30
+    PROFILE = Firefox()
31
+    city = URL('asset/t9/getLocalityT9.php\?site=fr&lang=fr&json=%22(?P<pattern>.*)%22',
32
+               CitiesPage)
33
+    search = URL('(?P<type>location-immobilier|vente-immobilier|recherche-colocation)-(?P<cities>.*)/options/(?P<options>.*)', SearchPage)
34
+    housing = URL('detail-(?P<_id>.*).htm', HousingPage)
35
+    phone = URL('(?P<urlcontact>.*)', PhonePage)
36
+
37
+    TYPES = {POSTS_TYPES.RENT: 'location-immobilier',
38
+             POSTS_TYPES.SALE: 'vente-immobilier',
39
+             POSTS_TYPES.SHARING: 'recherche-colocation',
40
+             POSTS_TYPES.FURNISHED_RENT: 'location-immobilier',
41
+             POSTS_TYPES.VIAGER: 'vente-immobilier'}
42
+
43
+    RET = {HOUSE_TYPES.HOUSE: '2',
44
+           HOUSE_TYPES.APART: '1',
45
+           HOUSE_TYPES.LAND: '3',
46
+           HOUSE_TYPES.PARKING: '10',
47
+           HOUSE_TYPES.OTHER: '14'}
48
+
49
+    def __init__(self, *args, **kwargs):
50
+        super(LogicimmoBrowser, self).__init__(*args, **kwargs)
51
+        self.session.headers['X-Requested-With'] = 'XMLHttpRequest'
52
+
53
+    def get_cities(self, pattern):
54
+        if pattern:
55
+            return self.city.go(pattern=pattern).get_cities()
56
+
57
+    def search_housings(self, type, cities, nb_rooms, area_min, area_max, cost_min, cost_max, house_types):
58
+        if type not in self.TYPES:
59
+            raise TypeNotSupported()
60
+
61
+        options = []
62
+
63
+        ret = []
64
+        if type == POSTS_TYPES.VIAGER:
65
+            ret = ['15']
66
+        else:
67
+            for house_type in house_types:
68
+                if house_type in self.RET:
69
+                    ret.append(self.RET.get(house_type))
70
+
71
+        if len(ret):
72
+            options.append('groupprptypesids=%s' % ','.join(ret))
73
+
74
+        if type == POSTS_TYPES.FURNISHED_RENT:
75
+            options.append('searchoptions=4')
76
+
77
+        options.append('pricemin=%s' % (cost_min if cost_min else '0'))
78
+
79
+        if cost_max:
80
+            options.append('pricemax=%s' % cost_max)
81
+
82
+        options.append('areamin=%s' % (area_min if area_min else '0'))
83
+
84
+        if area_max:
85
+            options.append('areamax=%s' % area_max)
86
+
87
+        if nb_rooms:
88
+            if type == POSTS_TYPES.SHARING:
89
+                options.append('nbbedrooms=%s' % ','.join([str(i) for i in range(nb_rooms, 7)]))
90
+            else:
91
+                options.append('nbrooms=%s' % ','.join([str(i) for i in range(nb_rooms, 7)]))
92
+
93
+        self.search.go(type=self.TYPES.get(type, 'location-immobilier'),
94
+                       cities=cities,
95
+                       options='/'.join(options))
96
+
97
+        if type == POSTS_TYPES.SHARING:
98
+            return self.page.iter_sharing()
99
+
100
+        return self.page.iter_housings(query_type=type)
101
+
102
+    def get_housing(self, _id, housing=None):
103
+        return self.housing.go(_id=_id).get_housing(obj=housing)
104
+
105
+    def get_phone(self, _id):
106
+        if _id.startswith('location') or _id.startswith('vente'):
107
+            urlcontact, params = self.housing.stay_or_go(_id=_id).get_phone_url_datas()
108
+            return self.phone.go(urlcontact=urlcontact, params=params).get_phone()

BIN
modules/logicimmo/favicon.png View File


+ 99
- 0
modules/logicimmo/module.py View File

@@ -0,0 +1,99 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+
21
+from weboob.tools.backend import Module
22
+from weboob.capabilities.housing import (CapHousing, Housing, HousingPhoto,
23
+                                         ADVERT_TYPES)
24
+from weboob.capabilities.base import UserError
25
+from .browser import LogicimmoBrowser
26
+
27
+
28
+__all__ = ['LogicimmoModule']
29
+
30
+
31
+class LogicImmoCitiesError(UserError):
32
+    """
33
+    Raised when more than 3 cities are selected
34
+    """
35
+    def __init__(self, msg='You cannot select more than three cities'):
36
+        UserError.__init__(self, msg)
37
+
38
+
39
+class LogicimmoModule(Module, CapHousing):
40
+    NAME = 'logicimmo'
41
+    DESCRIPTION = u'logicimmo website'
42
+    MAINTAINER = u'Bezleputh'
43
+    EMAIL = 'carton_ben@yahoo.fr'
44
+    LICENSE = 'AGPLv3+'
45
+    VERSION = '2.1'
46
+
47
+    BROWSER = LogicimmoBrowser
48
+
49
+    def get_housing(self, housing):
50
+        if isinstance(housing, Housing):
51
+            id = housing.id
52
+        else:
53
+            id = housing
54
+            housing = None
55
+        housing = self.browser.get_housing(id, housing)
56
+        return housing
57
+
58
+    def search_city(self, pattern):
59
+        return self.browser.get_cities(pattern)
60
+
61
+    def search_housings(self, query):
62
+        if(len(query.advert_types) == 1 and
63
+           query.advert_types[0] == ADVERT_TYPES.PERSONAL):
64
+            # Logic-immo is pro only
65
+            return list()
66
+
67
+        cities_names = ['%s' % c.name.replace(' ', '-') for c in query.cities if c.backend == self.name]
68
+        cities_ids = ['%s' % c.id for c in query.cities if c.backend == self.name]
69
+
70
+        if len(cities_names) == 0:
71
+            return list()
72
+
73
+        if len(cities_names) > 3:
74
+            raise LogicImmoCitiesError()
75
+
76
+        cities = ','.join(cities_names + cities_ids)
77
+        return self.browser.search_housings(query.type, cities.lower(), query.nb_rooms,
78
+                                            query.area_min, query.area_max,
79
+                                            query.cost_min, query.cost_max,
80
+                                            query.house_types)
81
+
82
+    def fill_housing(self, housing, fields):
83
+        if 'phone' in fields:
84
+            housing.phone = self.browser.get_phone(housing.id)
85
+            fields.remove('phone')
86
+
87
+        if len(fields) > 0:
88
+            self.browser.get_housing(housing.id, housing)
89
+
90
+        return housing
91
+
92
+    def fill_photo(self, photo, fields):
93
+        if 'data' in fields and photo.url and not photo.data:
94
+            photo.data = self.browser.open(photo.url).content
95
+        return photo
96
+
97
+    OBJECTS = {Housing: fill_housing,
98
+               HousingPhoto: fill_photo,
99
+               }

+ 377
- 0
modules/logicimmo/pages.py View File

@@ -0,0 +1,377 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright(C) 2014      Bezleputh
4
+#
5
+# This file is part of a weboob module.
6
+#
7
+# This weboob module is free software: you can redistribute it and/or modify
8
+# it under the terms of the GNU Affero General Public License as published by
9
+# the Free Software Foundation, either version 3 of the License, or
10
+# (at your option) any later version.
11
+#
12
+# This weboob module is distributed in the hope that it will be useful,
13
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+# GNU Affero General Public License for more details.
16
+#
17
+# You should have received a copy of the GNU Affero General Public License
18
+# along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
19
+
20
+from __future__ import unicode_literals
21
+
22
+from weboob.browser.pages import HTMLPage, JsonPage
23
+from weboob.browser.elements import ItemElement, ListElement, DictElement, method
24
+from weboob.browser.filters.json import Dict
25
+from weboob.browser.filters.standard import (Currency, Format, CleanText,
26
+                                             Regexp, CleanDecimal, Date, Env,
27
+                                             BrowserURL)
28
+from weboob.browser.filters.html import Attr, XPath, CleanHTML
29
+from weboob.capabilities.housing import (Housing, HousingPhoto, City,
30
+                                         UTILITIES, ENERGY_CLASS, POSTS_TYPES,
31
+                                         ADVERT_TYPES, HOUSE_TYPES)
32
+from weboob.capabilities.base import NotAvailable, NotLoaded
33
+from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
34
+from weboob.tools.compat import urljoin
35
+
36
+
37
+class CitiesPage(JsonPage):
38
+    @method
39
+    class get_cities(DictElement):
40
+        item_xpath = '*/children'
41
+
42
+        class item(ItemElement):
43
+            klass = City
44
+
45
+            def condition(self):
46
+                return Dict('lct_parent_id')(self) != '0'
47
+
48
+            obj_id = Format('%s_%s', Dict('lct_id'), Dict('lct_level'))
49
+            obj_name = Format('%s %s', Dict('lct_name'), Dict('lct_post_code'))
50
+
51
+
52
+class PhonePage(HTMLPage):
53
+    def get_phone(self):
54
+        return CleanText('//div[has-class("phone")]', children=False)(self.doc)
55
+
56
+
57
+class HousingPage(HTMLPage):
58
+    @method
59
+    class get_housing(ItemElement):
60
+        klass = Housing
61
+
62
+        obj_id = Env('_id')
63
+
64
+        def obj_type(self):
65
+            url = BrowserURL('housing', _id=Env('_id'))(self)
66
+            if 'colocation' in url:
67
+                return POSTS_TYPES.SHARING
68
+            elif 'location' in url:
69
+                isFurnished = False
70
+                for li in XPath('//ul[@itemprop="description"]/li')(self):
71
+                    label = CleanText('./span[has-class("criteria-label")]')(li)
72
+                    if label.lower() == "meublé":
73
+                        isFurnished = (
74
+                            CleanText('./span[has-class("criteria-value")]')(li).lower() == 'oui'
75
+                        )
76
+                if isFurnished:
77
+                    return POSTS_TYPES.FURNISHED_RENT
78
+                else:
79
+                    return POSTS_TYPES.RENT
80
+            elif 'vente' in url:
81
+                return POSTS_TYPES.SALE
82
+            return NotAvailable
83
+        obj_advert_type = ADVERT_TYPES.PROFESSIONAL
84
+
85
+        def obj_house_type(self):
86
+            house_type = CleanText('.//h2[@class="offerMainFeatures"]/div')(self).lower()
87
+            if house_type == "appartement":
88
+                return HOUSE_TYPES.APART
89
+            elif house_type == "maison":
90
+                return HOUSE_TYPES.HOUSE
91
+            elif house_type == "terrain":
92
+                return HOUSE_TYPES.LAND
93
+            elif house_type == "parking":
94
+                return HOUSE_TYPES.PARKING
95
+            else:
96
+                return HOUSE_TYPES.OTHER
97
+
98
+        obj_title = Attr('//meta[@property="og:title"]', 'content')
99
+        obj_area = CleanDecimal(
100
+            CleanText(
101
+                '//p[@class="offerArea"]/span',
102
+            ),
103
+            default=NotAvailable
104
+        )
105
+        obj_rooms = CleanDecimal(
106
+                        Regexp(
107
+                            CleanText('//p[@class="offerRooms"]/span'),
108
+                            '(\d) p.',
109
+                            default=NotAvailable
110
+                        ),
111
+                        default=NotAvailable
112
+                    )
113
+        obj_bedrooms = CleanDecimal(
114
+                        Regexp(
115
+                            CleanText('//p[@class="offerRooms"]/span'),
116
+                            '(\d) ch.',
117
+                            default=NotAvailable
118
+                        ),
119
+                        default=NotAvailable
120
+                    )
121
+        obj_cost = CleanDecimal('//*[@itemprop="price"]', default=0)
122
+        obj_currency = Currency(
123
+            '//*[@itemprop="price"]'
124
+        )
125
+
126
+        def obj_utilities(self):
127
+            notes = CleanText('//p[@class="offer-description-notes"]')(self)
128
+            if "Loyer mensuel charges comprises" in notes:
129
+                return UTILITIES.INCLUDED
130
+            else:
131
+                return UTILITIES.UNKNOWN
132
+
133
+        obj_price_per_meter = PricePerMeterFilter()
134
+        obj_date = Date(Regexp(CleanText('//div[@class="offer-description-notes"]'),
135
+                               u'.* Mis à jour: (\d{2}/\d{2}/\d{4}).*'),
136
+                        dayfirst=True)
137
+        obj_text = CleanHTML('//p[@class="descrProperty"]')
138
+        obj_location = CleanText('//em[@class="infoAdresse"]')
139
+        obj_station = CleanText(
140
+            '//div[has-class("offer-description-metro")]',
141
+            default=NotAvailable
142
+        )
143
+
144
+        obj_url = BrowserURL('housing', _id=Env('_id'))
145
+
146
+        def obj_photos(self):
147
+            photos = []
148
+            for img in XPath('//ul[@class="thumbsContainer"]//img/@src')(self):
149
+                if img.endswith('.svg'):
150
+                    continue
151
+                url = u'%s' % img.replace('182x136', '800x600')
152
+                url = urljoin(self.page.url, url)  # Ensure URL is absolute
153
+                photos.append(HousingPhoto(url))
154
+            return photos
155
+
156
+        def obj_DPE(self):
157
+            energy_value = CleanText(
158
+                '//ul[@class="energyInfosDPE"]//li[@class="energyInfos"]/span/@data-class',
159
+                default=""
160
+            )(self)
161
+            if len(energy_value):
162
+                energy_value = energy_value.replace("DPE", "").strip()[0]
163
+            return getattr(ENERGY_CLASS, energy_value, NotAvailable)
164
+
165
+        def obj_GES(self):
166
+            greenhouse_value = CleanText(
167
+                '//ul[@class="energyInfosGES"]//li[@class="energyInfos"]/span/@data-class',
168
+                default=""
169
+            )(self)
170
+            if len(greenhouse_value):
171
+                greenhouse_value = greenhouse_value.replace("GES", "").strip()[0]
172
+            return getattr(ENERGY_CLASS, greenhouse_value, NotAvailable)
173
+
174
+        def obj_details(self):
175
+            details = {}
176
+
177
+            details["creationDate"] = Date(
178
+                Regexp(
179
+                    CleanText(
180
+                        '//div[@class="offer-description-notes"]'
181
+                    ),
182
+                    u'.*Mis en ligne: (\d{2}/\d{2}/\d{4}).*'
183
+                ),
184
+                dayfirst=True
185
+            )(self)
186
+
187
+            honoraires = CleanText(
188
+                (
189
+                    '//div[has-class("offer-price")]/span[has-class("lbl-agencyfees")]'
190
+                ),
191
+                default=None
192
+            )(self)
193
+            if honoraires:
194
+                details["Honoraires"] = (
195
+                    "{} (TTC, en sus)".format(
196
+                        honoraires.split(":")[1].strip()
197
+                    )
198
+                )
199
+
200
+            for li in XPath('//ul[@itemprop="description"]/li')(self):
201
+                label = CleanText('./span[has-class("criteria-label")]')(li)
202
+                value = CleanText('./span[has-class("criteria-value")]')(li)
203
+                details[label] = value
204
+
205
+            return details
206
+
207
+    def get_phone_url_datas(self):
208
+        a = XPath('//button[has-class("js-show-phone-offer-sale-bottom")]')(self.doc)[0]
209
+        urlcontact = 'http://www.logic-immo.com/modalMail'
210
+        params = {}
211
+        params['universe'] = CleanText('./@data-univers')(a)
212
+        params['source'] = CleanText('./@data-source')(a)
213
+        params['pushcontact'] = CleanText('./@data-pushcontact')(a)
214
+        params['mapper'] = CleanText('./@data-mapper')(a)
215
+        params['offerid'] = CleanText('./@data-offerid')(a)
216
+        params['offerflag'] = CleanText('./@data-offerflag')(a)
217
+        params['campaign'] = CleanText('./@data-campaign')(a)
218
+        params['xtpage'] = CleanText('./@data-xtpage')(a)
219
+        params['offertransactiontype'] = CleanText('./@data-offertransactiontype')(a)
220
+        params['aeisource'] = CleanText('./@data-aeisource')(a)
221
+        params['shownumber'] = CleanText('./@data-shownumber')(a)
222
+        params['corail'] = 1
223
+        return urlcontact, params
224
+
225
+
226
+class SearchPage(HTMLPage):
227
+    @method
228
+    class iter_sharing(ListElement):
229
+        item_xpath = '//article[has-class("offer-block")]'
230
+
231
+        class item(ItemElement):
232
+            klass = Housing
233
+
234
+            obj_id = Format('colocation-%s', CleanText('./div/header/@id', replace=[('header-offer-', '')]))
235
+            obj_type = POSTS_TYPES.SHARING
236
+            obj_advert_type = ADVERT_TYPES.PROFESSIONAL
237
+            obj_title = CleanText(CleanHTML('./div/header/section/p[@class="property-type"]/span/@title'))
238
+
239
+            obj_area = CleanDecimal('./div/header/section/p[@class="offer-attributes"]/a/span[@class="offer-area-number"]',
240
+                                    default=0)
241
+
242
+            obj_cost = CleanDecimal('./div/header/section/p[@class="price"]', default=0)
243
+            obj_currency = Currency(
244
+                './div/header/section/p[@class="price"]'
245
+            )
246
+            obj_utilities = UTILITIES.UNKNOWN
247
+
248
+            obj_text = CleanText(
249
+                './div/div[@class="content-offer"]/section[has-class("content-desc")]/p/span[has-class("offer-text")]/@title',
250
+                default=NotLoaded
251
+            )
252
+
253
+            obj_date = Date(Regexp(CleanText('./div/header/section/p[has-class("update-date")]'),
254
+                                   ".*(\d{2}/\d{2}/\d{4}).*"))
255
+
256
+            obj_location = CleanText(
257
+                '(./div/div[@class="content-offer"]/section[has-class("content-desc")]/p)[1]/span/@title',
258
+                default=NotLoaded
259
+            )
260
+
261
+    @method
262
+    class iter_housings(ListElement):
263
+        item_xpath = '//div[has-class("offer-list")]//div[has-class("offer-block")]'
264
+
265
+        class item(ItemElement):
266
+            offer_details_wrapper = (
267
+                './/div[has-class("offer-details-wrapper")]'
268
+            )
269
+            klass = Housing
270
+
271
+            obj_id = Format(
272
+                '%s-%s',
273
+                Regexp(Env('type'), '(.*)-.*'),
274
+                CleanText('./@id', replace=[('header-offer-', '')])
275
+            )
276
+            obj_type = Env('query_type')
277
+            obj_advert_type = ADVERT_TYPES.PROFESSIONAL
278
+
279
+            def obj_house_type(self):
280
+                house_type = CleanText('.//div[has-class("offer-details-caracteristik")]/meta[@itemprop="name"]/@content')(self).lower()
281
+                if house_type == "appartement":
282
+                    return HOUSE_TYPES.APART
283
+                elif house_type == "maison":
284
+                    return HOUSE_TYPES.HOUSE
285
+                elif house_type == "terrain":
286
+                    return HOUSE_TYPES.LAND
287
+                elif house_type == "parking":
288
+                    return HOUSE_TYPES.PARKING
289
+                else:
290
+                    return HOUSE_TYPES.OTHER
291
+
292
+            obj_title = CleanText('.//div[has-class("offer-details-type")]/a/@title')
293
+
294
+            obj_url = Format(u'%s%s',
295
+                             CleanText('.//div/a[@class="offer-link"]/@href'),
296
+                             CleanText('.//div/a[@class="offer-link"]/\
297
+@data-orpi', default=""))
298
+
299
+            obj_area = CleanDecimal(
300
+                (
301
+                    offer_details_wrapper +
302
+                    '/div/div/div[has-class("offer-details-second")]' +
303
+                    '/div/h3[has-class("offer-attributes")]/span' +
304
+                    '/span[has-class("offer-area-number")]'
305
+                ),
306
+                default=NotLoaded
307
+            )
308
+            obj_rooms = CleanDecimal(
309
+                (
310
+                    offer_details_wrapper +
311
+                    '/div/div/div[has-class("offer-details-second")]' +
312
+                    '/div/h3[has-class("offer-attributes")]' +
313
+                    '/span[has-class("offer-rooms")]' +
314
+                    '/span[has-class("offer-rooms-number")]'
315
+                ),
316
+                default=NotAvailable
317
+            )
318
+            obj_cost = CleanDecimal(
319
+                Regexp(
320
+                    CleanText(
321
+                        (