diff --git a/.babelrc b/.babelrc new file mode 100644 index 0000000..e8c66d1 --- /dev/null +++ b/.babelrc @@ -0,0 +1,4 @@ +{ + "presets": ["es2015", "stage-0"], + "plugins": ["transform-runtime"] +} diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..c2bac62 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,10 @@ +{ + extends: ["vue", /* your other extends */], + plugins: ["vue"], + "env": { + "browser": true + }, + rules: { + 'indent': ["error", 4, { 'SwitchCase': 1 }], + } +} diff --git a/.gitignore b/.gitignore index 5d0799b..93c23ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ build -*.json *.pyc *.swp *.swo *.db +config/ +node_modules +flatisfy/web/static/js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..df12db1 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,46 @@ +## TL;DR + +* Branch off `master`. +* One feature per commit. +* In case of changes request, amend your commit. + + +## Useful infos + +* There is a `hooks/pre-commit` file which can be used as a `pre-commit` git + hook to check coding style. +* Python coding style is PEP8. JS coding style is enforced by `eslint`. +* Some useful `npm` scripts are provided (`build` / `watch` / `lint`) + + +## Translating the webapp + +If you want to translate the webapp, just create a new folder in +`flatisfy/web/js_src/i18n` with the short name of your locale (typically, `en` +is for english). Copy the `flatisfy/web/js_src/i18n/en/index.js` file to this +new folder and translate the `messages` strings. + +Then, edit `flatisfy/web/js_src/i18n/index.js` file to include your new +locale. + + +## How to contribute + +* If you're thinking about a new feature, see if there's already an issue open + about it, or please open one otherwise. This will ensure that everybody is on + track for the feature and willing to see it in Flatisfy. +* One commit per feature. +* Branch off the `master ` branch. +* Check the linting of your code before doing a PR. +* Ideally, your merge-request should be mergeable without any merge commit, that + is, it should be a fast-forward merge. For this to happen, your code needs to + be always rebased onto `master`. Again, this is something nice to have that + I expect from recurring contributors, but not a big deal if you don't do it + otherwise. +* I'll look at it and might ask for a few changes. In this case, please create + new commits. When the final result looks good, I may ask you to squash the + WIP commits into a single one, to maintain the invariant of "one feature, one + commit". + + +Thanks! diff --git a/README.md b/README.md index b57b4c3..593814e 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,11 @@ The content of this repository is licensed under an MIT license, unless explicitly mentionned otherwise. +## Contributing + +See the `CONTRIBUTING.md` file for more infos. + + ## Thanks * [Weboob](http://weboob.org/) diff --git a/flatisfy/__main__.py b/flatisfy/__main__.py index 0641e16..2229173 100644 --- a/flatisfy/__main__.py +++ b/flatisfy/__main__.py @@ -90,6 +90,10 @@ def parse_args(argv=None): subparsers.add_parser("import", parents=[parent_parser], help="Import housing posts in database.") + # Purge subcommand parser + subparsers.add_parser("purge", parents=[parent_parser], + help="Purge database.") + # Serve subcommand parser parser_serve = subparsers.add_parser("serve", parents=[parent_parser], help="Serve the web app.") @@ -103,6 +107,7 @@ def main(): """ Main module code. """ + # pylint: disable=locally-disabled,too-many-branches # Parse arguments args = parse_args() @@ -163,7 +168,12 @@ def main(): ) # Import command elif args.cmd == "import": + # TODO: Do not fetch details for already imported flats / use the last + # timestamp cmds.import_and_filter(config) + # Purge command + elif args.cmd == "purge": + cmds.purge_db(config) # Serve command elif args.cmd == "serve": cmds.serve(config) diff --git a/flatisfy/cmds.py b/flatisfy/cmds.py index 08d7aee..f91678f 100644 --- a/flatisfy/cmds.py +++ b/flatisfy/cmds.py @@ -4,6 +4,8 @@ Main commands available for flatisfy. """ from __future__ import absolute_import, print_function, unicode_literals +import logging + import flatisfy.filters from flatisfy import database from flatisfy.models import flat as flat_model @@ -12,6 +14,9 @@ from flatisfy import tools from flatisfy.web import app as web_app +LOGGER = logging.getLogger(__name__) + + def fetch_and_filter(config): """ Fetch the available flats list. Then, filter it according to criteria. @@ -34,9 +39,9 @@ def fetch_and_filter(config): # additional infos if config["passes"] > 1: # Load additional infos - for flat in flats_list: - details = fetch.fetch_details(flat["id"]) - flat = tools.merge_dicts(flat, details) + for i, flat in enumerate(flats_list): + details = fetch.fetch_details(config, flat["id"]) + flats_list[i] = tools.merge_dicts(flat, details) flats_list, extra_ignored_flats = flatisfy.filters.second_pass( flats_list, config @@ -83,7 +88,7 @@ def import_and_filter(config): :return: ``None``. """ # Fetch and filter flats list - flats_list, purged_list = fetch_and_filter(config) + flats_list, ignored_list = fetch_and_filter(config) # Create database connection get_session = database.init_db(config["database"]) @@ -92,12 +97,27 @@ def import_and_filter(config): flat = flat_model.Flat.from_dict(flat_dict) session.merge(flat) - for flat_dict in purged_list: + for flat_dict in ignored_list: flat = flat_model.Flat.from_dict(flat_dict) - flat.status = flat_model.FlatStatus.purged + flat.status = flat_model.FlatStatus.ignored session.merge(flat) +def purge_db(config): + """ + Purge the database. + + :param config: A config dict. + :return: ``None`` + """ + get_session = database.init_db(config["database"]) + + with get_session() as session: + # Delete every flat in the db + LOGGER.info("Purge all flats from the database.") + session.query(flat_model.Flat).delete(synchronize_session=False) + + def serve(config): """ Serve the web app. @@ -106,5 +126,11 @@ def serve(config): :return: ``None``, long-running process. """ app = web_app.get_app(config) - # TODO: Make Bottle use logging module - app.run(host=config["host"], port=config["port"]) + + server = config.get("webserver", None) + if not server: + # Default webserver is quiet, as Bottle is used with Canister for + # standard logging + server = web_app.QuietWSGIRefServer + + app.run(host=config["host"], port=config["port"], server=server) diff --git a/flatisfy/config.py b/flatisfy/config.py index 35e694d..b9693ca 100644 --- a/flatisfy/config.py +++ b/flatisfy/config.py @@ -21,10 +21,11 @@ from flatisfy import tools # Default configuration DEFAULT_CONFIG = { - # Flatboob queries to fetch - "queries": [], # Constraints to match "constraints": { + "type": None, # RENT, SALE, SHARING + "house_types": [], # List of house types, must be in APART, HOUSE, + # PARKING, LAND, OTHER or UNKNOWN "postal_codes": [], # List of postal codes "area": (None, None), # (min, max) in m^2 "cost": (None, None), # (min, max) in currency unit @@ -42,12 +43,18 @@ DEFAULT_CONFIG = { "max_entries": None, # Directory in wich data will be put. ``None`` is XDG default location. "data_directory": None, + # Path to the modules directory containing all Weboob modules. ``None`` if + # ``weboob_modules`` package is pip-installed, and you want to use + # ``pkgresource`` to automatically find it. + "modules_path": None, # SQLAlchemy URI to the database to use "database": None, # Web app port "port": 8080, # Web app host to listen on - "host": "127.0.0.1" + "host": "127.0.0.1", + # Web server to use to serve the webapp (see Bottle deployment doc) + "webserver": None } LOGGER = logging.getLogger(__name__) @@ -68,7 +75,7 @@ def validate_config(config): assert all( x is None or ( - (isinstance(x, int) or isinstance(x, float)) and + isinstance(x, (float, int)) and x >= 0 ) for x in bounds @@ -81,9 +88,19 @@ def validate_config(config): # Then, we disable line-too-long pylint check and E501 flake8 checks # and use long lines whenever needed, in order to have the full assert # message in the log output. - # pylint: disable=line-too-long + # pylint: disable=locally-disabled,line-too-long + assert "type" in config["constraints"] + assert config["constraints"]["type"].upper() in ["RENT", + "SALE", "SHARING"] + + assert "house_types" in config["constraints"] + assert config["constraints"]["house_types"] + for house_type in config["constraints"]["house_types"]: + assert house_type.upper() in ["APART", "HOUSE", "PARKING", "LAND", + "OTHER", "UNKNOWN"] + assert "postal_codes" in config["constraints"] - assert len(config["constraints"]["postal_codes"]) > 0 + assert config["constraints"]["postal_codes"] assert "area" in config["constraints"] _check_constraints_bounds(config["constraints"]["area"]) @@ -111,11 +128,13 @@ def validate_config(config): assert config["max_entries"] is None or (isinstance(config["max_entries"], int) and config["max_entries"] > 0) # noqa: E501 assert config["data_directory"] is None or isinstance(config["data_directory"], str) # noqa: E501 + assert config["modules_path"] is None or isinstance(config["modules_path"], str) # noqa: E501 assert config["database"] is None or isinstance(config["database"], str) # noqa: E501 assert isinstance(config["port"], int) assert isinstance(config["host"], str) + assert config["webserver"] is None or isinstance(config["webserver"], str) # noqa: E501 return True except (AssertionError, KeyError): @@ -140,10 +159,11 @@ def load_config(args=None): try: with open(args.config, "r") as fh: config_data.update(json.load(fh)) - except (IOError, ValueError): + except (IOError, ValueError) as exc: LOGGER.error( "Unable to load configuration from file, " - "using default configuration." + "using default configuration: %s.", + exc ) # Overload config with arguments @@ -188,9 +208,8 @@ def load_config(args=None): if config_validation is True: LOGGER.info("Config has been fully initialized.") return config_data - else: - LOGGER.error("Error in configuration: %s.", config_validation) - return None + LOGGER.error("Error in configuration: %s.", config_validation) + return None def init_config(output=None): diff --git a/flatisfy/data.py b/flatisfy/data.py index 5f766c6..807cce8 100644 --- a/flatisfy/data.py +++ b/flatisfy/data.py @@ -9,6 +9,7 @@ import collections import json import logging import os +import shutil import flatisfy.exceptions @@ -157,7 +158,7 @@ def load_data(data_type, config): LOGGER.error("Invalid JSON data file: %s.", datafile_path) return None - if len(data) == 0: + if not data: LOGGER.warning("Loading empty data for %s.", data_type) return data diff --git a/flatisfy/database/__init__.py b/flatisfy/database/__init__.py index 06819a3..b23e1c7 100644 --- a/flatisfy/database/__init__.py +++ b/flatisfy/database/__init__.py @@ -41,16 +41,18 @@ def init_db(database_uri=None): engine = create_engine(database_uri) BASE.metadata.create_all(engine, checkfirst=True) - Session = sessionmaker(bind=engine) # pylint: disable=invalid-name + Session = sessionmaker(bind=engine) # pylint: disable=locally-disabled,invalid-name @contextmanager def get_session(): + # pylint: disable=locally-disabled,line-too-long """ Provide a transactional scope around a series of operations. From [1]. [1]: http://docs.sqlalchemy.org/en/latest/orm/session_basics.html#when-do-i-construct-a-session-when-do-i-commit-it-and-when-do-i-close-it. """ + # pylint: enable=line-too-long,locally-disabled session = Session() try: yield session diff --git a/flatisfy/database/types.py b/flatisfy/database/types.py index d55d7bb..ea2afdc 100644 --- a/flatisfy/database/types.py +++ b/flatisfy/database/types.py @@ -46,5 +46,5 @@ class StringyJSON(types.TypeDecorator): # TypeEngine.with_variant says "use StringyJSON instead when # connecting to 'sqlite'" -# pylint: disable=invalid-name +# pylint: disable=locally-disabled,invalid-name MagicJSON = types.JSON().with_variant(StringyJSON, 'sqlite') diff --git a/flatisfy/fetch.py b/flatisfy/fetch.py index addf2d3..7fea23b 100644 --- a/flatisfy/fetch.py +++ b/flatisfy/fetch.py @@ -4,14 +4,159 @@ This module contains all the code related to fetching and loading flats lists. """ from __future__ import absolute_import, print_function, unicode_literals +import itertools import json import logging -import subprocess +from flatisfy import data +from flatisfy import tools LOGGER = logging.getLogger(__name__) +try: + from weboob.capabilities.housing import Query + from weboob.core.ouiboube import WebNip + from weboob.tools.json import WeboobEncoder +except ImportError: + LOGGER.error("Weboob is not available on your system. Make sure you " + "installed it.") + raise + + +class WeboobProxy(object): + """ + Wrapper around Weboob ``WebNip`` class, to fetch housing posts without + having to spawn a subprocess. + """ + @staticmethod + def version(): + """ + Get Weboob version. + + :return: The installed Weboob version. + """ + return WebNip.VERSION + + def __init__(self, config): + """ + Create a Weboob handle and try to load the modules. + + :param config: A config dict. + """ + # Create base WebNip object + self.webnip = WebNip(modules_path=config["modules_path"]) + + # Create backends + self.backends = [ + self.webnip.load_backend( + module, + module, + params={} + ) + for module in ["seloger", "pap", "leboncoin", "logicimmo", + "explorimmo", "entreparticuliers"] + ] + + def __enter__(self): + return self + + def __exit__(self, *args): + self.webnip.deinit() + + def build_queries(self, constraints_dict): + """ + Build Weboob ``weboob.capabilities.housing.Query`` objects from the + constraints defined in the configuration. Each query has at most 3 + postal codes, to comply with housing websites limitations. + + :param constraints_dict: A dictionary of constraints, as defined in the + config. + :return: A list of Weboob ``weboob.capabilities.housing.Query`` + objects. Returns ``None`` if an error occurred. + """ + queries = [] + for postal_codes in tools.batch(constraints_dict["postal_codes"], 3): + query = Query() + query.cities = [] + for postal_code in postal_codes: + try: + for city in self.webnip.do("search_city", postal_code): + query.cities.append(city) + except IndexError: + LOGGER.error( + "Postal code %s could not be matched with a city.", + postal_code + ) + return None + + try: + query.house_types = [ + getattr( + Query.HOUSE_TYPES, + house_type.upper() + ) + for house_type in constraints_dict["house_types"] + ] + except AttributeError: + LOGGER.error("Invalid house types constraint.") + return None + + try: + query.type = getattr( + Query, + "TYPE_{}".format(constraints_dict["type"].upper()) + ) + except AttributeError: + LOGGER.error("Invalid post type constraint.") + return None + + query.area_min = constraints_dict["area"][0] + query.area_max = constraints_dict["area"][1] + query.cost_min = constraints_dict["cost"][0] + query.cost_max = constraints_dict["cost"][1] + query.nb_rooms = constraints_dict["rooms"][0] + + queries.append(query) + + return queries + + def query(self, query, max_entries=None): + """ + Fetch the housings posts matching a given Weboob query. + + :param query: A Weboob `weboob.capabilities.housing.Query`` object. + :param max_entries: Maximum number of entries to fetch. + :return: The matching housing posts, dumped as a list of JSON objects. + """ + housings = [] + # TODO: Handle max_entries better + for housing in itertools.islice( + self.webnip.do('search_housings', query), + max_entries + ): + housings.append(json.dumps(housing, cls=WeboobEncoder)) + return housings + + def info(self, full_flat_id): + """ + Get information (details) about an housing post. + + :param full_flat_id: A Weboob housing post id, in complete form + (ID@BACKEND) + :return: The details in JSON. + """ + flat_id, backend_name = full_flat_id.rsplit("@", 1) + backend = next( + backend + for backend in self.backends + if backend.name == backend_name + ) + housing = backend.get_housing(flat_id) + housing.id = full_flat_id # Otherwise, we miss the @backend afterwards + return json.dumps(housing, cls=WeboobEncoder) + + def fetch_flats_list(config): """ Fetch the available flats using the Flatboob / Weboob config. @@ -20,40 +165,35 @@ def fetch_flats_list(config): :return: A list of all available flats. """ flats_list = [] - for query in config["queries"]: - max_entries = config["max_entries"] - if max_entries is None: - max_entries = 0 - LOGGER.info("Loading flats from query %s.", query) - flatboob_output = subprocess.check_output( - ["../weboob/tools/local_run.sh", "../weboob/scripts/flatboob", - "-n", str(max_entries), "-f", "json", "load", query] - ) - query_flats_list = json.loads(flatboob_output) - LOGGER.info("Fetched %d flats.", len(query_flats_list)) - flats_list.extend(query_flats_list) - LOGGER.info("Fetched a total of %d flats.", len(flats_list)) + with WeboobProxy(config) as weboob_proxy: + LOGGER.info("Loading flats...") + queries = weboob_proxy.build_queries(config["constraints"]) + housing_posts = [] + for query in queries: + housing_posts.extend( + weboob_proxy.query(query, config["max_entries"]) + ) + LOGGER.info("Fetched %d flats.", len(housing_posts)) + + flats_list = [json.loads(flat) for flat in housing_posts] return flats_list -def fetch_details(flat_id): +def fetch_details(config, flat_id): """ Fetch the additional details for a flat using Flatboob / Weboob. + :param config: A config dict. :param flat_id: ID of the flat to fetch details for. :return: A flat dict with all the available data. """ - LOGGER.info("Loading additional details for flat %s.", flat_id) - flatboob_output = subprocess.check_output( - ["../weboob/tools/local_run.sh", "../weboob/scripts/flatboob", - "-f", "json", "info", flat_id] - ) - flat_details = json.loads(flatboob_output) - LOGGER.info("Fetched details for flat %s.", flat_id) + with WeboobProxy(config) as weboob_proxy: + LOGGER.info("Loading additional details for flat %s.", flat_id) + weboob_output = weboob_proxy.info(flat_id) - if flat_details: - flat_details = flat_details[0] + flat_details = json.loads(weboob_output) + LOGGER.info("Fetched details for flat %s.", flat_id) return flat_details diff --git a/flatisfy/filters/__init__.py b/flatisfy/filters/__init__.py index 8addc68..2220a8b 100644 --- a/flatisfy/filters/__init__.py +++ b/flatisfy/filters/__init__.py @@ -89,9 +89,10 @@ def first_pass(flats_list, config): :param flats_list: A list of flats dict to filter. :param config: A config dict. - :return: A tuple of processed flats and purged flats. + :return: A tuple of processed flats and ignored flats. """ LOGGER.info("Running first filtering pass.") + # Handle duplicates based on ids # Just remove them (no merge) as they should be the exact same object. flats_list = duplicates.detect( @@ -105,16 +106,16 @@ def first_pass(flats_list, config): flats_list, key="url", merge=True ) - # Add the flatisfy metadata entry + # Add the flatisfy metadata entry and prepare the flat objects flats_list = metadata.init(flats_list) # Guess the postal codes flats_list = metadata.guess_postal_code(flats_list, config) # Try to match with stations flats_list = metadata.guess_stations(flats_list, config) # Remove returned housing posts that do not match criteria - flats_list, purged_list = refine_with_housing_criteria(flats_list, config) + flats_list, ignored_list = refine_with_housing_criteria(flats_list, config) - return (flats_list, purged_list) + return (flats_list, ignored_list) def second_pass(flats_list, config): @@ -130,7 +131,7 @@ def second_pass(flats_list, config): :param flats_list: A list of flats dict to filter. :param config: A config dict. - :return: A tuple of processed flats and purged flats. + :return: A tuple of processed flats and ignored flats. """ LOGGER.info("Running second filtering pass.") # Assumed to run after first pass, so there should be no obvious duplicates @@ -148,6 +149,6 @@ def second_pass(flats_list, config): flats_list = metadata.compute_travel_times(flats_list, config) # Remove returned housing posts that do not match criteria - flats_list, purged_list = refine_with_housing_criteria(flats_list, config) + flats_list, ignored_list = refine_with_housing_criteria(flats_list, config) - return (flats_list, purged_list) + return (flats_list, ignored_list) diff --git a/flatisfy/filters/duplicates.py b/flatisfy/filters/duplicates.py index 18f9264..a7365c3 100644 --- a/flatisfy/filters/duplicates.py +++ b/flatisfy/filters/duplicates.py @@ -5,9 +5,23 @@ Filtering functions to detect and merge duplicates. from __future__ import absolute_import, print_function, unicode_literals import collections +import logging from flatisfy import tools +LOGGER = logging.getLogger(__name__) + +# Some backends give more infos than others. Here is the precedence we want to +# use. +BACKENDS_PRECEDENCE = [ + "seloger", + "pap", + "leboncoin", + "explorimmo", + "logicimmo", + "entreparticuliers" +] + def detect(flats_list, key="id", merge=True): """ @@ -27,7 +41,6 @@ def detect(flats_list, key="id", merge=True): :return: A deduplicated list of flat dicts. """ - # TODO: Keep track of found duplicates? # ``seen`` is a dict mapping aggregating the flats by the deduplication # keys. We basically make buckets of flats for every key value. Flats in # the same bucket should be merged together afterwards. @@ -44,6 +57,18 @@ def detect(flats_list, key="id", merge=True): # of the others, to avoid over-deduplication. unique_flats_list.extend(matching_flats) else: + # Sort matching flats by backend precedence + matching_flats.sort( + key=lambda flat: next( + i for (i, backend) in enumerate(BACKENDS_PRECEDENCE) + if flat["id"].endswith(backend) + ), + reverse=True + ) + + if len(matching_flats) > 1: + LOGGER.info("Found duplicates: %s.", + [flat["id"] for flat in matching_flats]) # Otherwise, check the policy if merge: # If a merge is requested, do the merge diff --git a/flatisfy/filters/metadata.py b/flatisfy/filters/metadata.py index 002934d..e52050e 100644 --- a/flatisfy/filters/metadata.py +++ b/flatisfy/filters/metadata.py @@ -20,14 +20,20 @@ LOGGER = logging.getLogger(__name__) def init(flats_list): """ Create a flatisfy key containing a dict of metadata fetched by flatisfy for - each flat in the list. + each flat in the list. Also perform some basic transform on flat objects to + prepare for the metadata fetching. :param flats_list: A list of flats dict. :return: The updated list """ for flat in flats_list: + # Init flatisfy key if "flatisfy" not in flat: flat["flatisfy"] = {} + # Move url key to urls + flat["urls"] = [flat["url"]] + # Create merged_ids key + flat["merged_ids"] = [flat["id"]] return flats_list @@ -298,11 +304,17 @@ def guess_stations(flats_list, config, distance_threshold=1500): # If some stations were already filled in and the result is different, # display some warning to the user if ( - "matched_stations" in flat["flatisfy"]["matched_stations"] and + "matched_stations" in flat["flatisfy"] and ( # Do a set comparison, as ordering is not important - set(flat["flatisfy"]["matched_stations"]) != - set(good_matched_stations) + set([ + station["name"] + for station in flat["flatisfy"]["matched_stations"] + ]) != + set([ + station["name"] + for station in good_matched_stations + ]) ) ): LOGGER.warning( diff --git a/flatisfy/models/flat.py b/flatisfy/models/flat.py index f5d440d..12d2c74 100644 --- a/flatisfy/models/flat.py +++ b/flatisfy/models/flat.py @@ -2,9 +2,12 @@ """ This modules defines an SQLAlchemy ORM model for a flat. """ -# pylint: disable=invalid-name,too-few-public-methods +# pylint: disable=locally-disabled,invalid-name,too-few-public-methods from __future__ import absolute_import, print_function, unicode_literals +import logging + +import arrow import enum from sqlalchemy import Column, DateTime, Enum, Float, String, Text @@ -13,15 +16,29 @@ from flatisfy.database.base import BASE from flatisfy.database.types import MagicJSON +LOGGER = logging.getLogger(__name__) + + +class FlatUtilities(enum.Enum): + """ + An enum of the possible utilities status for a flat entry. + """ + included = 10 + unknown = 0 + excluded = -10 + + class FlatStatus(enum.Enum): """ An enum of the possible status for a flat entry. """ - purged = -10 + user_deleted = -100 + ignored = -10 new = 0 - contacted = 10 - answer_no = 20 - answer_yes = 21 + followed = 10 + contacted = 20 + answer_no = 30 + answer_yes = 31 class Flat(BASE): @@ -36,6 +53,7 @@ class Flat(BASE): bedrooms = Column(Float) cost = Column(Float) currency = Column(String) + utilities = Column(Enum(FlatUtilities), default=FlatUtilities.unknown) date = Column(DateTime) details = Column(MagicJSON) location = Column(String) @@ -45,7 +63,8 @@ class Flat(BASE): station = Column(String) text = Column(Text) title = Column(String) - url = Column(String) + urls = Column(MagicJSON) + merged_ids = Column(MagicJSON) # Flatisfy data # TODO: Should be in another table with relationships @@ -65,25 +84,45 @@ class Flat(BASE): # Handle flatisfy metadata flat_dict = flat_dict.copy() flat_dict["flatisfy_stations"] = ( - flat_dict["flatisfy"].get("matched_stations", None) + flat_dict["flatisfy"].get("matched_stations", []) ) flat_dict["flatisfy_postal_code"] = ( flat_dict["flatisfy"].get("postal_code", None) ) flat_dict["flatisfy_time_to"] = ( - flat_dict["flatisfy"].get("time_to", None) + flat_dict["flatisfy"].get("time_to", {}) ) del flat_dict["flatisfy"] + # Handle utilities field + if not isinstance(flat_dict["utilities"], FlatUtilities): + if flat_dict["utilities"] == "C.C.": + flat_dict["utilities"] = FlatUtilities.included + elif flat_dict["utilities"] == "H.C.": + flat_dict["utilities"] = FlatUtilities.excluded + else: + flat_dict["utilities"] = FlatUtilities.unknown + + # Handle status field + flat_status = flat_dict.get("status", "new") + if not isinstance(flat_status, FlatStatus): + try: + flat_dict["status"] = getattr(FlatStatus, flat_status) + except AttributeError: + if "status" in flat_dict: + del flat_dict["status"] + LOGGER.warn("Unkown flat status %s, ignoring it.", + flat_status) + # Handle date field - flat_dict["date"] = None # TODO + flat_dict["date"] = arrow.get(flat_dict["date"]).naive flat_object = Flat() flat_object.__dict__.update(flat_dict) return flat_object def __repr__(self): - return "" % (self.id, self.url) + return "" % (self.id, self.urls) def json_api_repr(self): @@ -96,6 +135,9 @@ class Flat(BASE): for k, v in self.__dict__.items() if not k.startswith("_") } - flat_repr["status"] = str(flat_repr["status"]) + if isinstance(flat_repr["status"], FlatStatus): + flat_repr["status"] = flat_repr["status"].name + if isinstance(flat_repr["utilities"], FlatUtilities): + flat_repr["utilities"] = flat_repr["utilities"].name return flat_repr diff --git a/flatisfy/tools.py b/flatisfy/tools.py index f714286..ecc78e4 100644 --- a/flatisfy/tools.py +++ b/flatisfy/tools.py @@ -8,6 +8,7 @@ from __future__ import ( ) import datetime +import itertools import json import logging import math @@ -23,6 +24,16 @@ LOGGER = logging.getLogger(__name__) NAVITIA_ENDPOINT = "https://api.navitia.io/v1/coverage/fr-idf/journeys" +class DateAwareJSONEncoder(json.JSONEncoder): + """ + Extend the default JSON encoder to serialize datetimes to iso strings. + """ + def default(self, o): # pylint: disable=locally-disabled,E0202 + if isinstance(o, (datetime.date, datetime.datetime)): + return o.isoformat() + return json.JSONEncoder.default(self, o) + + def pretty_json(data): """ Pretty JSON output. @@ -38,10 +49,25 @@ def pretty_json(data): "toto": "ok" } """ - return json.dumps(data, indent=4, separators=(',', ': '), + return json.dumps(data, cls=DateAwareJSONEncoder, + indent=4, separators=(',', ': '), sort_keys=True) +def batch(iterable, size): + """ + Get items from a sequence a batch at a time. + + :param iterable: The iterable to get the items from. + :param size: The size of the batches. + :return: A new iterable. + """ + sourceiter = iter(iterable) + while True: + batchiter = itertools.islice(sourceiter, size) + yield itertools.chain([batchiter.next()], batchiter) + + def is_within_interval(value, min_value=None, max_value=None): """ Check whether a variable is within a given interval. Assumes the value is @@ -142,7 +168,7 @@ def distance(gps1, gps2): lat2 = math.radians(gps2[0]) long2 = math.radians(gps2[1]) - # pylint: disable=invalid-name + # pylint: disable=locally-disabled,invalid-name a = ( math.sin((lat2 - lat1) / 2.0)**2 + math.cos(lat1) * math.cos(lat2) * math.sin((long2 - long1) / 2.0)**2 @@ -175,22 +201,30 @@ def merge_dicts(*args): """ if len(args) == 1: return args[0] - else: - flat1, flat2 = args[:2] - merged_flat = {} - for k, value2 in flat2.items(): - value1 = flat1.get(k, None) - if value1 is None: - # flat1 has empty matching field, just keep the flat2 field - merged_flat[k] = value2 - elif value2 is None: - # flat2 field is empty, just keep the flat1 field - merged_flat[k] = value1 - else: - # Any other case, we should merge - # TODO: Do the merge - merged_flat[k] = value1 - return merge_dicts(merged_flat, *args[2:]) + + flat1, flat2 = args[:2] # pylint: disable=locally-disabled,unbalanced-tuple-unpacking,line-too-long + merged_flat = {} + for k, value2 in flat2.items(): + value1 = flat1.get(k, None) + + if k in ["urls", "merged_ids"]: + # Handle special fields separately + merged_flat[k] = list(set(value2 + value1)) + continue + + if not value1: + # flat1 has empty matching field, just keep the flat2 field + merged_flat[k] = value2 + elif not value2: + # flat2 field is empty, just keep the flat1 field + merged_flat[k] = value1 + else: + # Any other case, we should keep the value of the more recent flat + # dict (the one most at right in arguments) + merged_flat[k] = value2 + for k in [key for key in flat1.keys() if key not in flat2.keys()]: + merged_flat[k] = flat1[k] + return merge_dicts(merged_flat, *args[2:]) def get_travel_time_between(latlng_from, latlng_to, config): diff --git a/flatisfy/web/app.py b/flatisfy/web/app.py index 347bfb4..46dad4c 100644 --- a/flatisfy/web/app.py +++ b/flatisfy/web/app.py @@ -6,15 +6,30 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals ) +import functools +import json +import logging import os import bottle +import canister from flatisfy import database +from flatisfy.tools import DateAwareJSONEncoder from flatisfy.web.routes import api as api_routes +from flatisfy.web.configplugin import ConfigPlugin from flatisfy.web.dbplugin import DatabasePlugin +class QuietWSGIRefServer(bottle.WSGIRefServer): + """ + Quiet implementation of Bottle built-in WSGIRefServer, as `Canister` is + handling the logging through standard Python logging. + """ + # pylint: disable=locally-disabled,too-few-public-methods + quiet = True + + def _serve_static_file(filename): """ Helper function to serve static file. @@ -38,11 +53,31 @@ def get_app(config): app = bottle.default_app() app.install(DatabasePlugin(get_session)) + app.install(ConfigPlugin(config)) + app.config.setdefault("canister.log_level", logging.root.level) + app.config.setdefault("canister.log_path", None) + app.config.setdefault("canister.debug", False) + app.install(canister.Canister()) + # Use DateAwareJSONEncoder to dump JSON strings + # From http://stackoverflow.com/questions/21282040/bottle-framework-how-to-return-datetime-in-json-response#comment55718456_21282666. pylint: disable=locally-disabled,line-too-long + bottle.install( + bottle.JSONPlugin( + json_dumps=functools.partial(json.dumps, cls=DateAwareJSONEncoder) + ) + ) # API v1 routes app.route("/api/v1/", "GET", api_routes.index_v1) + + app.route("/api/v1/time_to/places", "GET", api_routes.time_to_places_v1) + app.route("/api/v1/flats", "GET", api_routes.flats_v1) + app.route("/api/v1/flats/status/:status", "GET", + api_routes.flats_by_status_v1) + app.route("/api/v1/flat/:flat_id", "GET", api_routes.flat_v1) + app.route("/api/v1/flat/:flat_id/status", "POST", + api_routes.update_flat_status_v1) # Index app.route("/", "GET", lambda: _serve_static_file("index.html")) diff --git a/flatisfy/web/configplugin.py b/flatisfy/web/configplugin.py new file mode 100644 index 0000000..68d8a04 --- /dev/null +++ b/flatisfy/web/configplugin.py @@ -0,0 +1,72 @@ +# coding: utf-8 +""" +This module contains a Bottle plugin to pass the config argument to any route +which needs it. + +This module is heavily based on code from +[Bottle-SQLAlchemy](https://github.com/iurisilvio/bottle-sqlalchemy) which is +licensed under MIT license. +""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals +) + +import functools +import inspect + +import bottle + + +class ConfigPlugin(object): + """ + A Bottle plugin to automatically pass the config object to the routes + specifying they need it. + """ + name = 'config' + api = 2 + KEYWORD = "config" + + def __init__(self, config): + """ + :param config: The config object to pass. + """ + self.config = config + + def setup(self, app): # pylint: disable=no-self-use + """ + Make sure that other installed plugins don't affect the same + keyword argument and check if metadata is available. + """ + for other in app.plugins: + if not isinstance(other, ConfigPlugin): + continue + else: + raise bottle.PluginError( + "Found another conflicting Config plugin." + ) + + def apply(self, callback, route): + """ + Method called on route invocation. Should apply some transformations to + the route prior to returing it. + + We check the presence of ``self.KEYWORD`` in the route signature and + replace the route callback by a partial invocation where we replaced + this argument by a valid config object. + """ + # Check whether the route needs a valid db session or not. + try: + callback_args = inspect.signature(route.callback).parameters + except AttributeError: + # inspect.signature does not exist on older Python + callback_args = inspect.getargspec(route.callback).args + + if self.KEYWORD not in callback_args: + # If no need for a db session, call the route callback + return callback + kwargs = {} + kwargs[self.KEYWORD] = self.config + return functools.partial(callback, **kwargs) + + +Plugin = ConfigPlugin diff --git a/flatisfy/web/dbplugin.py b/flatisfy/web/dbplugin.py index 6b5e0b8..1049b9b 100644 --- a/flatisfy/web/dbplugin.py +++ b/flatisfy/web/dbplugin.py @@ -28,13 +28,12 @@ class DatabasePlugin(object): def __init__(self, get_session): """ - :param keyword: Keyword used to inject session database in a route :param create_session: SQLAlchemy session maker created with the 'sessionmaker' function. Will create its own if undefined. """ self.get_session = get_session - def setup(self, app): # pylint: disable-no-self-use + def setup(self, app): # pylint: disable=no-self-use """ Make sure that other installed plugins don't affect the same keyword argument and check if metadata is available. @@ -67,11 +66,15 @@ class DatabasePlugin(object): # If no need for a db session, call the route callback return callback else: - # Otherwise, we get a db session and pass it to the callback - with self.get_session() as session: - kwargs = {} - kwargs[self.KEYWORD] = session - return functools.partial(callback, **kwargs) + def wrapper(*args, **kwargs): + """ + Wrap the callback in a call to get_session. + """ + with self.get_session() as session: + # Get a db session and pass it to the callback + kwargs[self.KEYWORD] = session + return callback(*args, **kwargs) + return wrapper Plugin = DatabasePlugin diff --git a/flatisfy/web/js_src/api/index.js b/flatisfy/web/js_src/api/index.js new file mode 100644 index 0000000..3b54f42 --- /dev/null +++ b/flatisfy/web/js_src/api/index.js @@ -0,0 +1,63 @@ +import moment from 'moment' + +require('es6-promise').polyfill() +require('isomorphic-fetch') + +export const getFlats = function (callback) { + fetch('/api/v1/flats') + .then(function (response) { + return response.json() + }).then(function (json) { + const flats = json.data + flats.map(flat => { + if (flat.date) { + flat.date = moment(flat.date) + } + return flat + }) + callback(flats) + }).catch(function (ex) { + console.error('Unable to parse flats: ' + ex) + }) +} + +export const getFlat = function (flatId, callback) { + fetch('/api/v1/flat/' + encodeURIComponent(flatId)) + .then(function (response) { + return response.json() + }).then(function (json) { + const flat = json.data + if (flat.date) { + flat.date = moment(flat.date) + } + callback(json.data) + }).catch(function (ex) { + console.error('Unable to parse flats: ' + ex) + }) +} + +export const updateFlatStatus = function (flatId, newStatus, callback) { + fetch( + '/api/v1/flat/' + encodeURIComponent(flatId) + '/status', + { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + status: newStatus + }) + } + ).then(callback) +} + +export const getTimeToPlaces = function (callback) { + fetch('/api/v1/time_to/places') + .then(function (response) { + return response.json() + }).then(function (json) { + callback(json.data) + }).catch(function (ex) { + console.error('Unable to fetch time to places: ' + ex) + }) +} diff --git a/flatisfy/web/js_src/components/app.vue b/flatisfy/web/js_src/components/app.vue new file mode 100644 index 0000000..c147d32 --- /dev/null +++ b/flatisfy/web/js_src/components/app.vue @@ -0,0 +1,76 @@ + + + + + diff --git a/flatisfy/web/js_src/components/flatsmap.vue b/flatisfy/web/js_src/components/flatsmap.vue new file mode 100644 index 0000000..69b155d --- /dev/null +++ b/flatisfy/web/js_src/components/flatsmap.vue @@ -0,0 +1,103 @@ + + + + + + + diff --git a/flatisfy/web/js_src/components/flatstable.vue b/flatisfy/web/js_src/components/flatstable.vue new file mode 100644 index 0000000..f714900 --- /dev/null +++ b/flatisfy/web/js_src/components/flatstable.vue @@ -0,0 +1,90 @@ + + + + + diff --git a/flatisfy/web/js_src/components/slider.vue b/flatisfy/web/js_src/components/slider.vue new file mode 100644 index 0000000..eec3c32 --- /dev/null +++ b/flatisfy/web/js_src/components/slider.vue @@ -0,0 +1,144 @@ + + + + + diff --git a/flatisfy/web/js_src/i18n/en/index.js b/flatisfy/web/js_src/i18n/en/index.js new file mode 100644 index 0000000..5aa0603 --- /dev/null +++ b/flatisfy/web/js_src/i18n/en/index.js @@ -0,0 +1,55 @@ +export default { + common: { + 'flats': 'flat | flats', + 'loading': 'Loading…', + 'Actions': 'Actions', + 'More': 'More', + 'Remove': 'Remove', + 'Restore': 'Restore', + 'External_link': 'External link', + 'Follow': 'Follow', + 'Close': 'Close' + }, + home: { + 'new_available_flats': 'New available flats' + }, + flatListing: { + 'no_available_flats': 'No available flats.' + }, + menu: { + 'available_flats': 'Available flats', + 'followed_flats': 'Followed flats', + 'ignored_flats': 'Ignored flats', + 'user_deleted_flats': 'User deleted flats' + }, + flatsDetails: { + 'Title': 'Title', + 'Area': 'Area', + 'Rooms': 'Rooms', + 'Cost': 'Cost', + 'utilities_included': '(utilities included)', + 'utilities_excluded': '(utilities excluded)', + 'Description': 'Description', + 'Details': 'Details', + 'Metadata': 'Metadata', + 'postal_code': 'Postal code', + 'nearby_stations': 'Nearby stations', + 'Times_to': 'Times to', + 'Location': 'Location', + 'Contact': 'Contact', + 'no_phone_found': 'No phone found', + 'Original_posts': 'Original posts:', + 'Original_post': 'Original post', + 'rooms': 'room | rooms', + 'bedrooms': 'bedroom | bedrooms' + }, + status: { + 'new': 'new', + 'followed': 'followed', + 'ignored': 'ignored', + 'user_deleted': 'user deleted' + }, + slider: { + 'Fullscreen_photo': 'Fullscreen photo' + } +} diff --git a/flatisfy/web/js_src/i18n/index.js b/flatisfy/web/js_src/i18n/index.js new file mode 100644 index 0000000..9f0f33e --- /dev/null +++ b/flatisfy/web/js_src/i18n/index.js @@ -0,0 +1,52 @@ +import Vue from 'vue' +import VueI18n from 'vue-i18n' + +// Import translations +import en from './en' + +Vue.use(VueI18n) + +export function getBrowserLocales () { + let langs = [] + + if (navigator.languages) { + // Chrome does not currently set navigator.language correctly + // https://code.google.com/p/chromium/issues/detail?id=101138 + // but it does set the first element of navigator.languages correctly + langs = navigator.languages + } else if (navigator.userLanguage) { + // IE only + langs = [navigator.userLanguage] + } else { + // as of this writing the latest version of firefox + safari set this correctly + langs = [navigator.language] + } + + // Some browsers does not return uppercase for second part + const locales = langs.map(function (lang) { + const locale = lang.split('-') + return locale[1] ? `${locale[0]}-${locale[1].toUpperCase()}` : lang + }) + + return locales +} + +const messages = { + 'en': en +} + +const locales = getBrowserLocales() + +var locale = 'en' // Safe default +// Get best matching locale +for (var i = 0; i < locales.length; ++i) { + if (messages[locales[i]]) { + locale = locales[i] + break // Break at first matching locale + } +} + +export default new VueI18n({ + locale: locale, + messages +}) diff --git a/flatisfy/web/js_src/main.js b/flatisfy/web/js_src/main.js new file mode 100644 index 0000000..7c8fcd0 --- /dev/null +++ b/flatisfy/web/js_src/main.js @@ -0,0 +1,14 @@ +import Vue from 'vue' + +import i18n from './i18n' +import router from './router' +import store from './store' + +import App from './components/app.vue' + +new Vue({ + i18n, + router, + store, + render: createEle => createEle(App) +}).$mount('#app') diff --git a/flatisfy/web/js_src/router/index.js b/flatisfy/web/js_src/router/index.js new file mode 100644 index 0000000..627a459 --- /dev/null +++ b/flatisfy/web/js_src/router/index.js @@ -0,0 +1,18 @@ +import Vue from 'vue' +import VueRouter from 'vue-router' + +import Home from '../views/home.vue' +import Status from '../views/status.vue' +import Details from '../views/details.vue' + +Vue.use(VueRouter) + +export default new VueRouter({ + routes: [ + { path: '/', component: Home, name: 'home' }, + { path: '/followed', component: Status, name: 'followed' }, + { path: '/ignored', component: Status, name: 'ignored' }, + { path: '/user_deleted', component: Status, name: 'user_deleted' }, + { path: '/flat/:id', component: Details, name: 'details' } + ] +}) diff --git a/flatisfy/web/js_src/store/actions.js b/flatisfy/web/js_src/store/actions.js new file mode 100644 index 0000000..cb2ed09 --- /dev/null +++ b/flatisfy/web/js_src/store/actions.js @@ -0,0 +1,26 @@ +import * as api from '../api' +import * as types from './mutations-types' + +export default { + getAllFlats ({ commit }) { + api.getFlats(flats => { + commit(types.REPLACE_FLATS, { flats }) + }) + }, + getFlat ({ commit }, { flatId }) { + api.getFlat(flatId, flat => { + const flats = [flat] + commit(types.MERGE_FLATS, { flats }) + }) + }, + getAllTimeToPlaces ({ commit }) { + api.getTimeToPlaces(timeToPlaces => { + commit(types.RECEIVE_TIME_TO_PLACES, { timeToPlaces }) + }) + }, + updateFlatStatus ({ commit }, { flatId, newStatus }) { + api.updateFlatStatus(flatId, newStatus, response => { + commit(types.UPDATE_FLAT_STATUS, { flatId, newStatus }) + }) + } +} diff --git a/flatisfy/web/js_src/store/getters.js b/flatisfy/web/js_src/store/getters.js new file mode 100644 index 0000000..914bd38 --- /dev/null +++ b/flatisfy/web/js_src/store/getters.js @@ -0,0 +1,56 @@ +import { findFlatGPS } from '../tools' + +export default { + allFlats: state => state.flats, + + flat: (state, getters) => id => state.flats.find(flat => flat.id === id), + + postalCodesFlatsBuckets: (state, getters) => filter => { + const postalCodeBuckets = {} + + state.flats.forEach(flat => { + if (filter && filter(flat)) { + const postalCode = flat.flatisfy_postal_code.postal_code + if (!postalCodeBuckets[postalCode]) { + postalCodeBuckets[postalCode] = { + 'name': flat.flatisfy_postal_code.name, + 'flats': [] + } + } + postalCodeBuckets[postalCode].flats.push(flat) + } + }) + + return postalCodeBuckets + }, + + flatsMarkers: (state, getters) => (router, filter) => { + const markers = [] + state.flats.forEach(flat => { + if (filter && filter(flat)) { + const gps = findFlatGPS(flat) + + if (gps) { + const previousMarkerIndex = markers.findIndex( + marker => marker.gps[0] === gps[0] && marker.gps[1] === gps[1] + ) + + const href = router.resolve({ name: 'details', params: { id: flat.id }}).href + if (previousMarkerIndex !== -1) { + markers[previousMarkerIndex].content += '
' + flat.title + '' + } else { + markers.push({ + 'title': '', + 'content': '' + flat.title + '', + 'gps': gps + }) + } + } + } + }) + + return markers + }, + + allTimeToPlaces: state => state.timeToPlaces +} diff --git a/flatisfy/web/js_src/store/index.js b/flatisfy/web/js_src/store/index.js new file mode 100644 index 0000000..7220393 --- /dev/null +++ b/flatisfy/web/js_src/store/index.js @@ -0,0 +1,16 @@ +import Vue from 'vue' +import Vuex from 'vuex' + +import actions from './actions' +import getters from './getters' +import { state, mutations } from './mutations' +// import products from './modules/products' + +Vue.use(Vuex) + +export default new Vuex.Store({ + state, + actions, + getters, + mutations +}) diff --git a/flatisfy/web/js_src/store/mutations-types.js b/flatisfy/web/js_src/store/mutations-types.js new file mode 100644 index 0000000..c2138b5 --- /dev/null +++ b/flatisfy/web/js_src/store/mutations-types.js @@ -0,0 +1,4 @@ +export const REPLACE_FLATS = 'REPLACE_FLATS' +export const MERGE_FLATS = 'MERGE_FLATS' +export const UPDATE_FLAT_STATUS = 'UPDATE_FLAT_STATUS' +export const RECEIVE_TIME_TO_PLACES = 'RECEIVE_TIME_TO_PLACES' diff --git a/flatisfy/web/js_src/store/mutations.js b/flatisfy/web/js_src/store/mutations.js new file mode 100644 index 0000000..4c7604b --- /dev/null +++ b/flatisfy/web/js_src/store/mutations.js @@ -0,0 +1,34 @@ +import Vue from 'vue' + +import * as types from './mutations-types' + +export const state = { + flats: [], + timeToPlaces: [] +} + +export const mutations = { + [types.REPLACE_FLATS] (state, { flats }) { + state.flats = flats + }, + [types.MERGE_FLATS] (state, { flats }) { + flats.forEach(flat => { + const flatIndex = state.flats.findIndex(storedFlat => storedFlat.id === flat.id) + + if (flatIndex > -1) { + Vue.set(state.flats, flatIndex, flat) + } else { + state.flats.push(flat) + } + }) + }, + [types.UPDATE_FLAT_STATUS] (state, { flatId, newStatus }) { + const index = state.flats.findIndex(flat => flat.id === flatId) + if (index > -1) { + Vue.set(state.flats[index], 'status', newStatus) + } + }, + [types.RECEIVE_TIME_TO_PLACES] (state, { timeToPlaces }) { + state.timeToPlaces = timeToPlaces + } +} diff --git a/flatisfy/web/js_src/tools/index.js b/flatisfy/web/js_src/tools/index.js new file mode 100644 index 0000000..1b041c4 --- /dev/null +++ b/flatisfy/web/js_src/tools/index.js @@ -0,0 +1,21 @@ +export function findFlatGPS (flat) { + let gps + + // Try to push a marker based on stations + if (flat.flatisfy_stations && flat.flatisfy_stations.length > 0) { + gps = [0.0, 0.0] + flat.flatisfy_stations.forEach(station => { + gps = [gps[0] + station.gps[0], gps[1] + station.gps[1]] + }) + gps = [gps[0] / flat.flatisfy_stations.length, gps[1] / flat.flatisfy_stations.length] + } else { + // Else, push a marker based on postal code + gps = flat.flatisfy_postal_code.gps + } + + return gps +} + +export function capitalize (string) { + return string.charAt(0).toUpperCase() + string.slice(1) +} diff --git a/flatisfy/web/js_src/views/details.vue b/flatisfy/web/js_src/views/details.vue new file mode 100644 index 0000000..d623e9a --- /dev/null +++ b/flatisfy/web/js_src/views/details.vue @@ -0,0 +1,275 @@ + + + + + diff --git a/flatisfy/web/js_src/views/home.vue b/flatisfy/web/js_src/views/home.vue new file mode 100644 index 0000000..d265100 --- /dev/null +++ b/flatisfy/web/js_src/views/home.vue @@ -0,0 +1,52 @@ + + + diff --git a/flatisfy/web/js_src/views/status.vue b/flatisfy/web/js_src/views/status.vue new file mode 100644 index 0000000..03841e5 --- /dev/null +++ b/flatisfy/web/js_src/views/status.vue @@ -0,0 +1,41 @@ + + + diff --git a/flatisfy/web/routes/api.py b/flatisfy/web/routes/api.py index 8306740..a61c046 100644 --- a/flatisfy/web/routes/api.py +++ b/flatisfy/web/routes/api.py @@ -6,6 +6,11 @@ from __future__ import ( absolute_import, division, print_function, unicode_literals ) +import json + +import bottle + +import flatisfy.data from flatisfy.models import flat as flat_model @@ -20,28 +25,136 @@ def index_v1(): } -def flats_v1(db): +def flats_v1(config, db): """ API v1 flats route: GET /api/v1/flats + + :return: The available flats objects in a JSON ``data`` dict. """ + postal_codes = flatisfy.data.load_data("postal_codes", config) + flats = [ flat.json_api_repr() for flat in db.query(flat_model.Flat).all() ] + + for flat in flats: + if flat["flatisfy_postal_code"]: + postal_code_data = postal_codes[flat["flatisfy_postal_code"]] + flat["flatisfy_postal_code"] = { + "postal_code": flat["flatisfy_postal_code"], + "name": postal_code_data["nom"], + "gps": postal_code_data["gps"] + } + else: + flat["flatisfy_postal_code"] = {} + return { "data": flats } -def flat_v1(flat_id, db): +def flats_by_status_v1(status, db): + """ + API v1 flats route with a specific status: + + GET /api/v1/flats/status/:status + + :return: The matching flats objects in a JSON ``data`` dict. + """ + try: + flats = [ + flat.json_api_repr() + for flat in ( + db.query(flat_model.Flat) + .filter_by(status=getattr(flat_model.FlatStatus, status)) + .all() + ) + ] + except AttributeError: + return bottle.HTTPError(400, "Invalid status provided.") + + return { + "data": flats + } + + +def flat_v1(flat_id, config, db): """ API v1 flat route: GET /api/v1/flat/:flat_id + + :return: The flat object in a JSON ``data`` dict. + """ + postal_codes = flatisfy.data.load_data("postal_codes", config) + + flat = db.query(flat_model.Flat).filter_by(id=flat_id).first() + + if not flat: + return bottle.HTTPError(404, "No flat with id {}.".format(flat_id)) + + flat = flat.json_api_repr() + + if flat["flatisfy_postal_code"]: + postal_code_data = postal_codes[flat["flatisfy_postal_code"]] + flat["flatisfy_postal_code"] = { + "postal_code": flat["flatisfy_postal_code"], + "name": postal_code_data["nom"], + "gps": postal_code_data["gps"] + } + else: + flat["flatisfy_postal_code"] = {} + + return { + "data": flat + } + + +def update_flat_status_v1(flat_id, db): + """ + API v1 route to update flat status: + + POST /api/v1/flat/:flat_id/status + Data: { + "status": "NEW_STATUS" + } + + :return: The new flat object in a JSON ``data`` dict. """ flat = db.query(flat_model.Flat).filter_by(id=flat_id).first() + if not flat: + return bottle.HTTPError(404, "No flat with id {}.".format(flat_id)) + + try: + flat.status = getattr( + flat_model.FlatStatus, json.load(bottle.request.body)["status"] + ) + except (AttributeError, ValueError, KeyError): + return bottle.HTTPError(400, "Invalid status provided.") + + json_flat = flat.json_api_repr() + return { - "data": flat.json_api_repr() + "data": json_flat + } + + +def time_to_places_v1(config): + """ + API v1 route to fetch the details of the places to compute time to. + + GET /api/v1/time_to/places + + :return: The JSON dump of the places to compute time to (dict of places + names mapped to GPS coordinates). + """ + places = { + k: v["gps"] + for k, v in config["constraints"]["time_to"].items() + } + return { + "data": places } diff --git a/flatisfy/web/static/index.html b/flatisfy/web/static/index.html index 6c6f44c..00366cf 100644 --- a/flatisfy/web/static/index.html +++ b/flatisfy/web/static/index.html @@ -2,29 +2,13 @@ + Flatisfy - + + -
-

Flatisfy

- - - - - - - - - -
TitreLien
-
- +
+ diff --git a/flatisfy/web/static/js/9e9c77db241e8a58da99bf28694c907d.png b/flatisfy/web/static/js/9e9c77db241e8a58da99bf28694c907d.png new file mode 100644 index 0000000..33cf955 Binary files /dev/null and b/flatisfy/web/static/js/9e9c77db241e8a58da99bf28694c907d.png differ diff --git a/flatisfy/web/static/js/a159160a02a1caf189f5008c0d150f36.png b/flatisfy/web/static/js/a159160a02a1caf189f5008c0d150f36.png new file mode 100644 index 0000000..06a375f Binary files /dev/null and b/flatisfy/web/static/js/a159160a02a1caf189f5008c0d150f36.png differ diff --git a/flatisfy/web/static/js/c6477ef24ce2054017fce1a2d8800385.png b/flatisfy/web/static/js/c6477ef24ce2054017fce1a2d8800385.png new file mode 100644 index 0000000..45b0050 Binary files /dev/null and b/flatisfy/web/static/js/c6477ef24ce2054017fce1a2d8800385.png differ diff --git a/flatisfy/web/static/js/d3a5d64a8534322988a4bed1b7dbc8b0.png b/flatisfy/web/static/js/d3a5d64a8534322988a4bed1b7dbc8b0.png new file mode 100644 index 0000000..1116503 Binary files /dev/null and b/flatisfy/web/static/js/d3a5d64a8534322988a4bed1b7dbc8b0.png differ diff --git a/hooks/pre-commit b/hooks/pre-commit index 4404633..5509861 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -1,3 +1,4 @@ #!/bin/sh pylint --rcfile=.ci/pylintrc flatisfy +npm run lint diff --git a/package.json b/package.json new file mode 100644 index 0000000..677fc8f --- /dev/null +++ b/package.json @@ -0,0 +1,51 @@ +{ + "name": "Flatisfy", + "description": "Flatisfy is your new companion to ease your search of a new housing :)", + "author": "Phyks (Lucas Verney) ", + "license": "MIT", + "version": "0.0.1", + "repository": { + "type": "git", + "url": "https://Phyks@git.phyks.me/Phyks/flatisfy.git" + }, + "homepage": "https://git.phyks.me/Phyks/flatisfy", + "scripts": { + "build": "webpack --colors --progress", + "watch": "webpack --colors --progress --watch", + "lint": "eslint --ext .js,.vue ./flatisfy/web/js_src/**" + }, + "dependencies": { + "es6-promise": "^4.1.0", + "imagesloaded": "^4.1.1", + "isomorphic-fetch": "^2.2.1", + "isotope-layout": "^3.0.3", + "leaflet.icon.glyph": "^0.2.0", + "masonry": "0.0.2", + "moment": "^2.18.1", + "vue": "^2.2.6", + "vue-i18n": "^6.1.1", + "vue-images-loaded": "^1.1.2", + "vue-router": "^2.4.0", + "vue2-leaflet": "0.0.44", + "vueisotope": "^3.0.0-rc", + "vuex": "^2.3.0" + }, + "devDependencies": { + "babel-core": "^6.24.1", + "babel-loader": "^6.4.1", + "babel-plugin-transform-runtime": "^6.23.0", + "babel-preset-es2015": "^6.24.1", + "babel-preset-stage-0": "^6.24.1", + "css-loader": "^0.28.0", + "eslint": "^3.19.0", + "eslint-config-vue": "^2.0.2", + "eslint-plugin-vue": "^2.0.1", + "file-loader": "^0.11.1", + "image-webpack-loader": "^3.3.0", + "style-loader": "^0.16.1", + "vue-html-loader": "^1.2.4", + "vue-loader": "^11.3.4", + "vue-template-compiler": "^2.2.6", + "webpack": "^2.3.3" + } +} diff --git a/requirements.txt b/requirements.txt index c74311d..8f32c51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ appdirs +arrow bottle bottle-sqlalchemy +canister enum34 future request diff --git a/webpack.config.js b/webpack.config.js new file mode 100644 index 0000000..054e6b2 --- /dev/null +++ b/webpack.config.js @@ -0,0 +1,55 @@ +module.exports = { + entry: './flatisfy/web/js_src/main.js', + output: { + path: __dirname + '/flatisfy/web/static/js/', + filename: 'bundle.js' + }, + module: { + loaders: [ + { + test: /\.js$/, + exclude: /(node_modules|bower_components)/, + use: { + loader: 'babel-loader' + } + }, + { + test: /\.vue$/, + loader: 'vue-loader', + options: { + loaders: { + js: 'babel-loader' + } + } + }, + { + test: /\.css$/, + loader: 'style-loader!css-loader' + }, + { + test: /\.(jpe?g|png|gif|svg)$/i, + loaders: [ + 'file-loader?hash=sha512&digest=hex&name=[hash].[ext]', + { + loader: 'image-webpack-loader', + query: { + bypassOnDebug: true, + 'optipng': { + optimizationLevel: 7 + }, + 'gifsicle': { + interlaced: false + } + } + } + ] + } + ] + }, + resolve: { + alias: { + 'masonry': 'masonry-layout', + 'isotope': 'isotope-layout' + } + } +} diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 0000000..fa9d6ac --- /dev/null +++ b/wsgi.py @@ -0,0 +1,33 @@ +# coding: utf-8 +""" +Expose a WSGI-compatible application to serve with a webserver. +""" +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import os +import sys + +import flatisfy.config +from flatisfy.web import app as web_app + + +class Args(): + config = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "config/config.json" + ) + + +LOGGER = logging.getLogger("flatisfy") + + +CONFIG = flatisfy.config.load_config(Args()) +if CONFIG is None: + LOGGER.error("Invalid configuration. Exiting. " + "Run init-config before if this is the first time " + "you run Flatisfy.") + sys.exit(1) + + +application = app = web_app.get_app(CONFIG)