diff --git a/.gitignore b/.gitignore index 93c23ac..8850236 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ build *.db config/ node_modules -flatisfy/web/static/js +flatisfy/web/static/assets diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df12db1..fa80f27 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,8 @@ * There is a `hooks/pre-commit` file which can be used as a `pre-commit` git hook to check coding style. * Python coding style is PEP8. JS coding style is enforced by `eslint`. -* Some useful `npm` scripts are provided (`build` / `watch` / `lint`) +* Some useful `npm` scripts are provided (`build:{dev,prod}` / + `watch:{dev,prod}` / `lint`) ## Translating the webapp diff --git a/README.md b/README.md index 593814e..c00d50c 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,12 @@ Flatisfy Flatisfy is your new companion to ease your search of a new housing :) +**Note**: This software is under heavy development at the moment, and the +database schema could change at any time. Do not consider it as being +production ready. However, I am currently using it for my own housing search +and it is working fine :) + + It uses [Weboob](http://weboob.org/) to get all the housing posts on most of the websites offering housings posts, and then offers a bunch of pipelines to filter and deduplicate the fetched housings. @@ -29,60 +35,20 @@ This code is not restricted to handling flats only! 1. Clone the repository. 2. Install required Python modules: `pip install -r requirements.txt`. 3. Init a configuration file: `python -m flatisfy init-config > config.json`. - Edit it according to your needs (see below). + Edit it according to your needs (see doc). 4. Build the required data files: `python -m flatisfy build-data --config config.json`. 5. Use it to `fetch` (and output a filtered JSON list of flats) or `import` (into an SQLite database, for the web visualization) a list of flats matching your criteria. -6. Use `python -m flatisfy serve --config config.json` to serve the web app. +6. Install JS libraries and build the webapp: + `npm install && npm run build:dev` (use `build:prod` in production). +7. Use `python -m flatisfy serve --config config.json` to serve the web app. -## Configuration +## Documentation -List of configuration options: - -* `data_directory` is the directory in which you want data files to be stored. - `null` is the default value and means default `XDG` location (typically - `~/.local/share/flatisfy/`) -* `max_entries` is the maximum number of entries to fetch **per Weboob - backend** (that is per housing website). -* `passes` is the number of passes to run on the data. First pass is a basic - filtering and using only the informations from the housings list page. - Second pass loads any possible information about the filtered flats and does - better filtering. -* `queries` is a list of queries defined in `flatboob` that should be fetched. -* `database` is an SQLAlchemy URI to a database file. Defaults to `null` which - means that it will store the database in the default location, in - `data_directory`. -* `navitia_api_key` is an API token for [Navitia](https://www.navitia.io/) - which is required to compute travel times. - -### Constraints - -You can specify constraints, under the `constraints` key. The available -constraints are: - -* `area` (in m²), `bedrooms`, `cost` (in currency unit), `rooms`: this is a - tuple of `(min, max)` values, defining an interval in which the value should - lie. A `null` value means that any value is within this bound. -* `postal_codes` is a list of allowed postal codes. You should include any - postal code you want, and especially the postal codes close to the precise - location you want. You MUST provide some postal codes. -* `time_to` is a dictionary of places to compute travel time to them. - Typically, - ``` - "time_to": { - "foobar": { - "gps": [LAT, LNG], - "time": [min, max] - } - } - ``` - means that the housings must be between the `min` and `max` bounds (possibly - `null`) from the place identified by the GPS coordinates `LAT` and `LNG` - (latitude and longitude), and we call this place `foobar` in human-readable - form. Beware that `time` constraints are in **seconds**. +See the [dedicated folder](doc/). ## OpenData diff --git a/doc/0.getting_started.md b/doc/0.getting_started.md new file mode 100644 index 0000000..1953fce --- /dev/null +++ b/doc/0.getting_started.md @@ -0,0 +1,106 @@ +Getting started +=============== + +## TL;DR + +1. Clone the repository. +2. Install required Python modules: `pip install -r requirements.txt`. +3. Init a configuration file: `python -m flatisfy init-config > config.json`. + Edit it according to your needs (see below). +4. Build the required data files: + `python -m flatisfy build-data --config config.json`. +5. Use it to `fetch` (and output a filtered JSON list of flats) or `import` + (into an SQLite database, for the web visualization) a list of flats + matching your criteria. +6. Install JS libraries and build the webapp: + `npm install && npm run build:dev` (use `build:prod` in production). +7. Use `python -m flatisfy serve --config config.json` to serve the web app. + + +## Available commands + +The available commands are: + +* `init-config` to generate an empty configuration file, either on the `stdin` + or in the specified file. +* `fetch` to load and filter housings posts and output a JSON dump. +* `filter` to filter a previously fetched list of housings posts, provided as + a JSON dump. +* `import` to import and filter housing posts into the database. +* `serve` to serve the built-in webapp with the development server. Do not use + in production. + + +## Configuration + +List of configuration options: + +* `data_directory` is the directory in which you want data files to be stored. + `null` is the default value and means default `XDG` location (typically + `~/.local/share/flatisfy/`) +* `max_entries` is the maximum number of entries to fetch. +* `passes` is the number of passes to run on the data. First pass is a basic + filtering and using only the informations from the housings list page. + Second pass loads any possible information about the filtered flats and does + better filtering. +* `database` is an SQLAlchemy URI to a database file. Defaults to `null` which + means that it will store the database in the default location, in + `data_directory`. +* `navitia_api_key` is an API token for [Navitia](https://www.navitia.io/) + which is required to compute travel times. +* `modules_path` is the path to the Weboob modules. It can be `None` if you + want Weboob to use the locally pip-installed modules (default value). +* `port` is the port on which the development webserver should be + listening (default to `8080`). +* `host` is the host on which the development webserver should be listening + (default to `127.0.0.1`). +* `webserver` is a server to use instead of the default Bottle built-in + webserver, see [Bottle deployment + doc](http://bottlepy.org/docs/dev/deployment.html). + +_Note:_ In production, you can either use the `serve` command with a reliable +webserver instead of the default Bottle webserver (specifying a `webserver` +value) or use the `wsgi.py` script at the root of the repository to use WSGI. + + +### Constraints + +You should specify some constraints to filter the resulting housings list, +under the `constraints` key. The available constraints are: + +* `type` is the type of housing you want, either `RENT` (to rent), `SALE` (to + buy) or `SHARING` (for a shared housing). +* `housing_types` is a list of house types you are looking for. Values can be + `APART` (flat), `HOUSE`, `PARKING`, `LAND`, `OTHER` (everything else) or + `UNKNOWN` (anything which was not matched with one of the previous + categories). +* `area` (in m²), `bedrooms`, `cost` (in currency unit), `rooms`: this is a + tuple of `(min, max)` values, defining an interval in which the value should + lie. A `null` value means that any value is within this bound. +* `postal_codes` is a list of postal codes. You should include any postal code + you want, and especially the postal codes close to the precise location you + want. +* `time_to` is a dictionary of places to compute travel time to them. + Typically, + + ``` + "time_to": { + "foobar": { + "gps": [LAT, LNG], + "time": [min, max] + } + } + ``` + + means that the housings must be between the `min` and `max` bounds (possibly + `null`) from the place identified by the GPS coordinates `LAT` and `LNG` + (latitude and longitude), and we call this place `foobar` in human-readable + form. Beware that `time` constraints are in **seconds**. + + +## Building the web assets + +If you want to build the web assets, you can use `npm run build:dev` +(respectively `npm run watch:dev` to build continuously and monitor changes in +source files). You can use `npm run build:prod` (`npm run watch:prod`) to do +the same in production mode (with minification etc). diff --git a/doc/1.production.md b/doc/1.production.md new file mode 100644 index 0000000..a8df4cf --- /dev/null +++ b/doc/1.production.md @@ -0,0 +1,223 @@ +Use in production environment +============================= + +This is the guide for the manual installation of a "production" instance. + +**IMPORTANT**: At the moment, there is **no** authentication mechanism in +Flatisfy. That is, if you want your instance to be private, you should put +some access control on your own with the webserver. This is explained below in +the case of `Nginx`. + +## Get the app installed + +```bash +# Clone the repository +git clone https://Phyks@git.phyks.me/Phyks/flatisfy.git && cd flatisfy + +# Create a virtualenv +virtualenv .env && source .env/bin/activate + +# Install required Python modules +pip install -r requirements.txt + +# Clone and install weboob +git clone https://git.weboob.org/weboob/devel weboob && cd weboob && python setup.py install && cd .. + +# Install required JS libraries and build the webapp +npm install && npm run build:prod + +# Create a minimal config file +mkdir config && python -m flatisfy init-config > config/config.json + +# Create a data directory to store db and data files +mkdir data + +# Edit the config file according to your needs +$EDITOR config/config.json + +# Build the required data files +python -m flatisfy build-data --config config/config.json -v + +# Run initial import +python -m flatisfy import --config config/config.json -v +``` + +_Note_: In the config, you should set `data_directory` to the absolute path of +the `data` directory created below. This directory should be writable by the +user running Flatisfy. You should also set `modules_path` to the absolute path +to the `modules` folder under the previous `weboob` clone. Finally, the last +`import` command can be `cron`-tasked to automatically fetch available +housings posts periodically. + + +## Use an alternative Bottle backend (production) + +This is the simplest option, offers good performances but is less scalable. +You should take care of which user is running `flatisfy`, as there can be some +related security concerns. + +Just choose another [available +webserver](https://bottlepy.org/docs/dev/deployment.html) to use in Bottle +(typically `cherrypi`) and set the `webserver` config option accordingly to +use it. + +### Nginx vhost + +Here is a typical minimal `Nginx` vhost you can use (which provides +HTTP authentication support and SSL): + +``` +upstream _flatisfy { + server 127.0.0.1:PORT; +} + +server { + listen 443; + server_name SERVER_NAME; + root ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT/flatisfy/web/static; + + access_log /var/log/nginx/flatisfy-access.log; + error_log /var/log/nginx/flatisfy-error.log warn; + + ssl on; + ssl_certificate PATH_TO_SSL_CERT; + ssl_certificate_key PATH_TO_SSL_KEY; + + location / { + auth_basic "Restricted"; + auth_basic_user_file ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT/.htpasswd; + + # Try to serve directly the URI first (static content), fallback on + # passing it to the Python backend + try_files $uri @uwsgi; + } + + location @uwsgi { + proxy_pass http://_flatisfy; + } +} + + +server { + listen 80; + server_name SERVER_NAME; + + root /dev/null; + + return 301 https://$server_name$request_uri; +} +``` + +where `PORT` (default for Flatisfy is 8080), `SERVER_NAME`, +`ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT`, `PATH_TO_SSL_CERT` and `PATH_TO_SSL_KEY` +should be replaced by values according to your own setup. You should also set +the `.htpasswd` file with users and credentials. + +_Note_: This vhost is really minimalistic and you should adapt it to your +setup, enforce SSL ciphers for increased security and do such good practices +things. + + +## Use WSGI + +This is the best option in terms of performance and is recommended. It should +offer both the best security and performances. + +### Configure uWSGI + +Assuming you are running Debian stable, you should install `uwsgi`: + +``` +apt-get install uwsgi uwsgi-plugin-python +``` + +Then, you can create a `/etc/uwsgi/apps-available/flatisfy.ini` with the +following content: + +```ini +[uwsgi] +socket = /run/uwsgi/app/flatisfy/socket +chdir = ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT +master = true +plugins = python +venv = ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT/.env +file = wsgi.py +uid = www-data +gid = www-data +``` + +where `ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT` is the absolute path to the root of +the Flatisfy git clone made at the beginning of this document. + +Now, you can enable the app: + +```bash +ln -s /etc/uwsgi/apps-available/flatisfy.ini /etc/uwsgi/apps-enabled/flatisfy.ini +``` + +and restart `uWSGI`: + +```bash +systemctl restart uwsgi +``` + +_Note_: You should review the `wsgi.py` file at the root of the repository to +check it is matching you setup (and especially is loading the configuration +from the right place, which is `config/config.json` by default). + + +### Nginx vhost + +Here is a typical minimal `Nginx` vhost you can use (which provides +HTTP authentication support and SSL): + +``` +upstream _flatisfy { + server unix:/run/uwsgi/app/flatisfy/socket; +} + +server { + listen 443; + server_name SERVER_NAME; + root ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT/flatisfy/web/static; + + access_log /var/log/nginx/flatisfy-access.log; + error_log /var/log/nginx/flatisfy-error.log warn; + + ssl on; + ssl_certificate PATH_TO_SSL_CERT; + ssl_certificate_key PATH_TO_SSL_KEY; + + location / { + auth_basic "Restricted"; + auth_basic_user_file ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT/.htpasswd; + + # Try to serve directly the URI first (static content), fallback on + # passing it to the Python backend + try_files $uri @uwsgi; + } + + location @uwsgi { + include uwsgi_params; + uwsgi_pass _flatisfy; + } +} + + +server { + listen 80; + server_name SERVER_NAME; + + root /dev/null; + + return 301 https://$server_name$request_uri; +} +``` + +where `SERVER_NAME`, `ABSOLUTE_PATH_TO_FLATISFY_GIT_ROOT`, `PATH_TO_SSL_CERT` +and `PATH_TO_SSL_KEY` should be replaced by values according to your own +setup. You should also set the `.htpasswd` file with users and credentials. + +_Note_: This vhost is really minimalistic and you should adapt it to your +setup, enforce SSL ciphers for increased security and do such good practices +things. diff --git a/flatisfy/web/app.py b/flatisfy/web/app.py index 46dad4c..220d426 100644 --- a/flatisfy/web/app.py +++ b/flatisfy/web/app.py @@ -83,6 +83,9 @@ def get_app(config): app.route("/", "GET", lambda: _serve_static_file("index.html")) # Static files - app.route("/static/", "GET", _serve_static_file) + app.route( + "/assets/", "GET", + lambda filename: _serve_static_file("/assets/{}".format(filename)) + ) return app diff --git a/flatisfy/web/js_src/components/flatsmap.vue b/flatisfy/web/js_src/components/flatsmap.vue index 69b155d..d8d9465 100644 --- a/flatisfy/web/js_src/components/flatsmap.vue +++ b/flatisfy/web/js_src/components/flatsmap.vue @@ -42,9 +42,9 @@ export default { }, icons: { flat: L.icon({ - iconUrl: '/static/js/' + markerUrl, - iconRetinaUrl: '/static/js' + marker2XUrl, - shadowUrl: '/static/js' + shadowUrl + iconUrl: markerUrl, + iconRetinaUrl: marker2XUrl, + shadowUrl: shadowUrl }), place: L.icon.glyph({ prefix: 'fa', diff --git a/flatisfy/web/js_src/components/flatstable.vue b/flatisfy/web/js_src/components/flatstable.vue index f714900..0512a6d 100644 --- a/flatisfy/web/js_src/components/flatstable.vue +++ b/flatisfy/web/js_src/components/flatstable.vue @@ -52,6 +52,8 @@ + diff --git a/flatisfy/web/static/js/9e9c77db241e8a58da99bf28694c907d.png b/flatisfy/web/static/js/9e9c77db241e8a58da99bf28694c907d.png deleted file mode 100644 index 33cf955..0000000 Binary files a/flatisfy/web/static/js/9e9c77db241e8a58da99bf28694c907d.png and /dev/null differ diff --git a/flatisfy/web/static/js/a159160a02a1caf189f5008c0d150f36.png b/flatisfy/web/static/js/a159160a02a1caf189f5008c0d150f36.png deleted file mode 100644 index 06a375f..0000000 Binary files a/flatisfy/web/static/js/a159160a02a1caf189f5008c0d150f36.png and /dev/null differ diff --git a/flatisfy/web/static/js/c6477ef24ce2054017fce1a2d8800385.png b/flatisfy/web/static/js/c6477ef24ce2054017fce1a2d8800385.png deleted file mode 100644 index 45b0050..0000000 Binary files a/flatisfy/web/static/js/c6477ef24ce2054017fce1a2d8800385.png and /dev/null differ diff --git a/flatisfy/web/static/js/d3a5d64a8534322988a4bed1b7dbc8b0.png b/flatisfy/web/static/js/d3a5d64a8534322988a4bed1b7dbc8b0.png deleted file mode 100644 index 1116503..0000000 Binary files a/flatisfy/web/static/js/d3a5d64a8534322988a4bed1b7dbc8b0.png and /dev/null differ diff --git a/package.json b/package.json index 677fc8f..c5f8bee 100644 --- a/package.json +++ b/package.json @@ -10,12 +10,16 @@ }, "homepage": "https://git.phyks.me/Phyks/flatisfy", "scripts": { - "build": "webpack --colors --progress", - "watch": "webpack --colors --progress --watch", + "build:dev": "webpack --colors --progress", + "watch:dev": "webpack --colors --progress --watch", + "build:prod": "NODE_ENV=production webpack --colors --progress -p", + "watch:prod": "NODE_ENV=production webpack --colors --progress --watch -p", "lint": "eslint --ext .js,.vue ./flatisfy/web/js_src/**" }, "dependencies": { "es6-promise": "^4.1.0", + "font-awesome": "^4.7.0", + "font-awesome-webpack": "0.0.5-beta.2", "imagesloaded": "^4.1.1", "isomorphic-fetch": "^2.2.1", "isotope-layout": "^3.0.3", @@ -42,7 +46,9 @@ "eslint-plugin-vue": "^2.0.1", "file-loader": "^0.11.1", "image-webpack-loader": "^3.3.0", + "less": "^2.7.2", "style-loader": "^0.16.1", + "url-loader": "^0.5.8", "vue-html-loader": "^1.2.4", "vue-loader": "^11.3.4", "vue-template-compiler": "^2.2.6", diff --git a/webpack.config.js b/webpack.config.js index 054e6b2..20e447c 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,8 +1,9 @@ module.exports = { entry: './flatisfy/web/js_src/main.js', output: { - path: __dirname + '/flatisfy/web/static/js/', - filename: 'bundle.js' + path: __dirname + '/flatisfy/web/static/assets/', + filename: 'bundle.js', + publicPath: '/assets/' }, module: { loaders: [ @@ -43,6 +44,14 @@ module.exports = { } } ] + }, + { + test: /\.woff(2)?(\?v=[0-9]\.[0-9]\.[0-9])?$/, + loader: "url-loader?limit=10000&mimetype=application/font-woff" + }, + { + test: /\.(ttf|eot|svg)(\?v=[0-9]\.[0-9]\.[0-9])?$/, + loader: "file-loader" } ] },