Flatisfy is your new companion to ease your search of a new housing :)

pages.py 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301
  1. # -*- coding: utf-8 -*-
  2. # Copyright(C) 2014 Bezleputh
  3. #
  4. # This file is part of a weboob module.
  5. #
  6. # This weboob module is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU Affero General Public License as published by
  8. # the Free Software Foundation, either version 3 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # This weboob module is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU Affero General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Affero General Public License
  17. # along with this weboob module. If not, see <http://www.gnu.org/licenses/>.
  18. from __future__ import unicode_literals
  19. import requests
  20. from weboob.browser.pages import HTMLPage, JsonPage, pagination
  21. from weboob.browser.elements import ItemElement, ListElement, method, DictElement
  22. from weboob.capabilities.base import Currency as BaseCurrency
  23. from weboob.browser.filters.standard import (CleanText, CleanDecimal, _Filter,
  24. Env, DateTime, Format)
  25. from weboob.browser.filters.json import Dict
  26. from weboob.capabilities.housing import (City, Housing, HousingPhoto,
  27. UTILITIES, ENERGY_CLASS, POSTS_TYPES,
  28. ADVERT_TYPES, HOUSE_TYPES)
  29. from weboob.capabilities.base import NotAvailable
  30. from weboob.tools.capabilities.housing.housing import PricePerMeterFilter
  31. from decimal import Decimal
  32. from lxml import etree
  33. import json
  34. class PopDetail(_Filter):
  35. def __init__(self, name, default=NotAvailable):
  36. super(PopDetail, self).__init__(default)
  37. self.name = name
  38. def __call__(self, item):
  39. return item.env['details'].pop(self.name, self.default)
  40. class CityListPage(HTMLPage):
  41. def build_doc(self, content):
  42. content = super(CityListPage, self).build_doc(content)
  43. if content.getroot() is not None:
  44. return content
  45. return etree.Element("html")
  46. @method
  47. class get_cities(ListElement):
  48. item_xpath = '//li'
  49. class item(ItemElement):
  50. klass = City
  51. obj_id = Format('%s %s',
  52. CleanText('./span[has-class("city")]'),
  53. CleanText('./span[@class="zipcode"]'))
  54. obj_name = Format('%s %s',
  55. CleanText('./span[has-class("city")]'),
  56. CleanText('./span[@class="zipcode"]'))
  57. class HomePage(HTMLPage):
  58. def __init__(self, *args, **kwargs):
  59. HTMLPage.__init__(self, *args, **kwargs)
  60. add_content = CleanText('(//body/script)[4]', replace=[('window.FLUX_STATE = ', '')])(self.doc) or '{}'
  61. api_content = CleanText('(//body/script[@id="__NEXT_DATA__"])')(self.doc)
  62. self.htmldoc = self.doc
  63. self.api_content = json.loads(api_content)
  64. self.doc = json.loads(add_content)
  65. def get_api_key(self):
  66. return Dict('runtimeConfig/API/KEY')(self.api_content)
  67. class HousingListPage(JsonPage):
  68. def __init__(self, *args, **kwargs):
  69. JsonPage.__init__(self, *args, **kwargs)
  70. if 'ads' not in self.doc:
  71. self.doc['ads'] = []
  72. @pagination
  73. @method
  74. class get_housing_list(DictElement):
  75. item_xpath = 'ads'
  76. def next_page(self):
  77. data = Env('data')(self)
  78. if data['offset'] > self.page.doc['total_all']:
  79. return
  80. data['offset'] = data['offset'] + data['limit']
  81. return requests.Request("POST", self.page.url, data=json.dumps(data))
  82. class item(ItemElement):
  83. klass = Housing
  84. def parse(self, el):
  85. self.env['details'] = {obj['key']: obj['value_label'] for obj in self.el['attributes']}
  86. obj_id = Dict('list_id')
  87. obj_url = Dict('url')
  88. obj_type = Env('query_type')
  89. obj_area = CleanDecimal(PopDetail('square',
  90. default=0),
  91. default=NotAvailable)
  92. obj_rooms = CleanDecimal(PopDetail('rooms',
  93. default=0),
  94. default=NotAvailable)
  95. def obj_GES(self):
  96. ges = CleanText(PopDetail('ges', default='|'))(self)
  97. return getattr(ENERGY_CLASS, ges[0], NotAvailable)
  98. def obj_DPE(self):
  99. dpe = CleanText(PopDetail('energy_rate', default='|'))(self)
  100. return getattr(ENERGY_CLASS, dpe[0], NotAvailable)
  101. def obj_house_type(self):
  102. value = CleanText(PopDetail('real_estate_type'), default=' ')(self).lower()
  103. if value == 'parking':
  104. return HOUSE_TYPES.PARKING
  105. elif value == 'appartement':
  106. return HOUSE_TYPES.APART
  107. elif value == 'maison':
  108. return HOUSE_TYPES.HOUSE
  109. elif value == 'terrain':
  110. return HOUSE_TYPES.LAND
  111. else:
  112. return HOUSE_TYPES.OTHER
  113. def obj_utilities(self):
  114. value = CleanText(PopDetail('charges_included',
  115. default='Non'),
  116. default=NotAvailable)(self)
  117. if value == "Oui":
  118. return UTILITIES.INCLUDED
  119. else:
  120. return UTILITIES.EXCLUDED
  121. def obj_advert_type(self):
  122. line_pro = Dict('owner/type')(self)
  123. if line_pro == u'pro':
  124. return ADVERT_TYPES.PROFESSIONAL
  125. else:
  126. return ADVERT_TYPES.PERSONAL
  127. obj_title = Dict('subject')
  128. obj_cost = CleanDecimal(Dict('price/0', default=NotAvailable), default=Decimal(0))
  129. obj_currency = BaseCurrency.get_currency(u'€')
  130. obj_text = Dict('body')
  131. obj_location = Dict('location/city_label')
  132. obj_date = DateTime(Dict('first_publication_date'))
  133. def obj_photos(self):
  134. photos = []
  135. for img in Dict('images/urls_large', default=[])(self):
  136. photos.append(HousingPhoto(img))
  137. return photos
  138. def obj_type(self):
  139. try:
  140. breadcrumb = int(Dict('category_id')(self))
  141. except ValueError:
  142. breadcrumb = None
  143. if breadcrumb == 11:
  144. return POSTS_TYPES.SHARING
  145. elif breadcrumb == 10:
  146. isFurnished = CleanText(PopDetail('furnished', default=' '))(self)
  147. if isFurnished.lower() == u'meublé':
  148. return POSTS_TYPES.FURNISHED_RENT
  149. else:
  150. return POSTS_TYPES.RENT
  151. else:
  152. return POSTS_TYPES.SALE
  153. obj_price_per_meter = PricePerMeterFilter()
  154. obj_details = Env('details')
  155. class HousingPage(HomePage):
  156. def __init__(self, *args, **kwargs):
  157. HomePage.__init__(self, *args, **kwargs)
  158. self.doc = self.api_content["props"]["pageProps"]["ad"]
  159. def get_api_key(self):
  160. return Dict('runtimeConfig/API/KEY_JSON')(self.api_content)
  161. @method
  162. class get_housing(ItemElement):
  163. klass = Housing
  164. def parse(self, el):
  165. self.env['details'] = {obj['key']: obj['value_label'] for obj in el['attributes']}
  166. obj_id = Env('_id')
  167. obj_area = CleanDecimal(PopDetail('square',
  168. default=0),
  169. default=NotAvailable)
  170. obj_rooms = CleanDecimal(PopDetail('rooms',
  171. default=0),
  172. default=NotAvailable)
  173. def obj_GES(self):
  174. ges = CleanText(PopDetail('ges', default='|'))(self)
  175. return getattr(ENERGY_CLASS, ges[0], NotAvailable)
  176. def obj_DPE(self):
  177. dpe = CleanText(PopDetail('energy_rate', default='|'))(self)
  178. return getattr(ENERGY_CLASS, dpe[0], NotAvailable)
  179. def obj_house_type(self):
  180. value = CleanText(PopDetail('real_estate_type'), default=' ')(self).lower()
  181. if value == 'parking':
  182. return HOUSE_TYPES.PARKING
  183. elif value == 'appartement':
  184. return HOUSE_TYPES.APART
  185. elif value == 'maison':
  186. return HOUSE_TYPES.HOUSE
  187. elif value == 'terrain':
  188. return HOUSE_TYPES.LAND
  189. else:
  190. return HOUSE_TYPES.OTHER
  191. def obj_utilities(self):
  192. value = CleanText(PopDetail('charges_included',
  193. default='Non'),
  194. default=NotAvailable)(self)
  195. if value == "Oui":
  196. return UTILITIES.INCLUDED
  197. else:
  198. return UTILITIES.EXCLUDED
  199. obj_title = Dict('subject')
  200. obj_cost = CleanDecimal(Dict('price/0', default=NotAvailable), default=Decimal(0))
  201. obj_currency = BaseCurrency.get_currency(u'€')
  202. obj_text = Dict('body')
  203. obj_location = Dict('location/city_label')
  204. def obj_advert_type(self):
  205. line_pro = Dict('owner/type')(self)
  206. if line_pro == u'pro':
  207. return ADVERT_TYPES.PROFESSIONAL
  208. else:
  209. return ADVERT_TYPES.PERSONAL
  210. obj_date = DateTime(Dict('first_publication_date'))
  211. def obj_photos(self):
  212. photos = []
  213. for img in Dict('images/urls_large', default=[])(self):
  214. photos.append(HousingPhoto(img))
  215. return photos
  216. def obj_type(self):
  217. try:
  218. breadcrumb = int(Dict('category_id')(self))
  219. except ValueError:
  220. breadcrumb = None
  221. if breadcrumb == 11:
  222. return POSTS_TYPES.SHARING
  223. elif breadcrumb == 10:
  224. isFurnished = CleanText(PopDetail('furnished', default=' '))(self)
  225. if isFurnished.lower() == u'meublé':
  226. return POSTS_TYPES.FURNISHED_RENT
  227. else:
  228. return POSTS_TYPES.RENT
  229. else:
  230. return POSTS_TYPES.SALE
  231. obj_price_per_meter = PricePerMeterFilter()
  232. obj_url = Dict('url')
  233. obj_details = Env('details')
  234. class PhonePage(JsonPage):
  235. def get_phone(self):
  236. if Dict('utils/status')(self.doc) == u'OK':
  237. return Dict('utils/phonenumber')(self.doc)
  238. return NotAvailable