From 9beed09ed08c3f2b3f7bb0f4a0c31ba88cab5630 Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Tue, 16 Oct 2018 15:42:55 +0200 Subject: [PATCH] Improve documentation. Fix #45. --- CONTRIBUTING.md | 60 +++++++++++--- README.md | 116 ++------------------------- doc/0.hosting.md | 146 ++++++++++++++++++++++++++++++++++ doc/10.technical_notes.md | 26 ++++++ doc/20.api.md | 10 +++ scripts/api_doc.py | 20 +++++ server/routes.py | 134 ++++++++++++++++++++++++++++--- support/nginx/cyclassist.conf | 8 +- 8 files changed, 391 insertions(+), 129 deletions(-) create mode 100644 doc/0.hosting.md create mode 100644 doc/10.technical_notes.md create mode 100644 doc/20.api.md create mode 100755 scripts/api_doc.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3096c3f..50ec80b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,8 @@ well, if you are more comfortable speaking French. * In case of changes request, amend your commit. New issues and merge requests are better at -[Framagit](https://framagit.org/phyks/cyclassist). +[Framagit](https://framagit.org/phyks/cyclassist). If this is a blocking issue +for you, you can contribute through Github as well. You can contribute in many ways to Cycl'Assist, be it development, translations or simply communicating around about this webapp! @@ -43,12 +44,46 @@ python -m server to spawn the server-side part, listening on `localhost:8081`. +You might want to have a look at [`doc/0.hosting.md`](doc/0.hosting.md) for +extra information on the available configuration options and general guidance +with hosting the software. -## Useful scripts for dev -You can run `scripts/gps_to_gpx.py` on your GPX trace to create a -`src/tools/mock_gpx.json` file ready to be used as a mocking source for the -position data (just edit the `src/constants.js` file accordingly). +## Adding new opendata sources + +A few opendata files are already imported in Cycl'Assist. All the scripts to +fetch and import them are located under the `scripts/opendata` folder. + +If you find any other opendata file which is not already imported and would be +worth having in Cycl'Assist, please feel free to edit the scripts under +`scripts/opendata` and create a merge request! + + +## Useful tips for contributors + +### Mocking locations + +A useful tool for dev on Cycl'Assist is the ability to mock location (instead +of using the location provided by your GPS or manual position). This is +possible by editing the values `src/constants.js` file. You can either +generate random positions in a bounding box or rely on a GPX file. + +To mock position using a GPX file, there is one extra required step. You +should run `scripts/gps_to_gpx.py ` to create a file at +`src/tools/mock_gpx.json` which will be used as a mocking source +for the position data (provided the settings in `src/constants.js` require +mocking position using a GPX trace). + + +### _A la carte_ component + +We are using [_A la carte_](https://vuetifyjs.com/en/guides/a-la-carte) +Vuetify components to reduce the size of the build. + +If you then require new Vuetify components, check that they are indeed +included in the `src/vuetify.js` file. The `yarn list-vuetify-components` +command might be useful to help you determine which components are used across +the code. ## Translating @@ -56,8 +91,15 @@ position data (just edit the `src/constants.js` file accordingly). Translation is done directly on [Zanata](https://translate.zanata.org/iteration/view/cyclassist/master?dswid=7345). To add new strings to localize, edit the `src/i18n/en.json` file with your new -strings (and only this file). Then, you can run `yarn push-locales` to send -the updated locales to translate and `yarn pull-locales` to fetch the -translated files. To use these scripts you will need the -Translate-toolkit(`pip install translate-toolkit`) and the [Zanata Python CLI +strings (and only this file). English locale should be considered as the +reference locale and new strings should always be added there. + +Then, you can run `yarn push-locales` to send the updated locales to translate +to the Zanata server. + +Once the strings have been translated at Zanata, you can run `yarn +pull-locales` to fetch the translated files. + +_Note :_ To use these scripts you will need the Translate-toolkit(`pip install +translate-toolkit`) and the [Zanata Python CLI client](https://github.com/zanata/zanata-python-client). diff --git a/README.md b/README.md index de09191..8106bb5 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ serve the issues. This code is the code running behind https://cyclo.phyks.me/ **A sandbox instance if you want to try it out without polluting the main instance is available at https://cyclo-dev.phyks.me**. Please note however this is a -sandbox instance which might be ahead of the official https://cyclo.phyks.me -instance from time to time, so that it might break from time to time. +development instance which might be ahead of the official https://cyclo.phyks.me +instance from time to time. It might break from time to time. As of current version, only the client side part (code running on your local device) handles your geolocation. **Your precise geolocation is never sent** @@ -21,6 +21,7 @@ could know the location of the displayed map. + ## OpenData The data collected by https://cyclo.phyks.me/ is available under an @@ -31,119 +32,18 @@ https://cyclo.phyks.me/api/v1/reports. Statistics about the instance can be fetched at https://cyclo.phyks.me/api/v1/stats. -## Hosting your own -### Client part +## Documentation -#### Build setup - -``` bash -# Install JS dependencies -yarn install - -# Serve with hot reload at localhost:8080 -yarn dev - -# Build for production with minification -yarn build - -# Build for production and view the bundle analyzer report -yarn build --report -``` - -#### Useful environment variables - -You can pass a few environment variables to the `yarn build|dev` commands to -adapt the behavior to your needs. - -* `PUBLIC_PATH=https://.../foobar` to serve the app from a subdirectory. -* `API_BASE_URL=https://...` to specify the location of the server (defaults - to `/`). The value should end with a trailing slash. -* `THUNDERFOREST_API_KEY=` to pass an API key server to use for - [Thunderforest](http://thunderforest.com/) tiles (OpenCycleMap, etc). -* `API_TOKEN=` to pass a token required to access the server side API (check - below in the server part environment variables for more details). - -You should also have a look at the build variables under the `config/` -subdirectory. - - -#### Geographical extension - -While the frontend could theoretically work in the entire world without much -modifications, it is currently written with mainland France in mind, mostly -because that is the territory the authors are most familiar with. -Additionnally, this limits the volume of geographical data (such as OSM -extracts) to handle and makes managing the app easier. - -You could of course easily extend it to support other territories. The -French-specific parts of the code so far are: -* The [`AddressInput`](https://framagit.org/phyks/cyclassist/blob/master/src/components/AddressInput.vue) component which uses the [https://adresse.data.gouv.fr/](https://adresse.data.gouv.fr/) API to autocomplete addresses. You could easily replace it with [Algolia Places](https://community.algolia.com/places/) which covers the entire world. - - -#### Notes - -We are using [A la carte](https://vuetifyjs.com/en/guides/a-la-carte) Vuetify -components to reduce the size of the build. Check that any extra components -you might use is indeed included in `src/vuetify.js` file. The `yarn -list-vuetify-components` command might be useful to help you determine which -components are used across the code. - - -### Server part - -#### Build setup - -``` bash -# Install Python dependencies -pip install -r requirements.txt - -# Start the server -python -m server -``` - -It is better to use a dedicated `virtualenv` if you can :) - -API routes are all listed within `server/routes.py` file, with documentation -strings. - -#### Useful environment variables - -You can pass a few environment variables to the `python -m server` command to -adapt its behavior: - -* `HOST=` to specify the host to listen to (defaults to `127.0.0.1` which - means `localhost` only). -* `PORT=` to specify the port to listen on (defaults to `8081`). -* `DATABASE=` to specify a [database URL](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#db-url) to connect to (defaults to - `sqlite:///reports.db` which means a SQLite database named `reports.db` in - the current working directory). -* `API_TOKEN=` to specify a token required to `POST` data to the API. - -#### Serving in production - -You can use the `wsgi.py` script at the root of the git repository to serve -the server side part. You can find some `uwsgi` and `nginx` base config files -under the `support` folder. - -#### Importing OpenData - -A few OpenData files can be imported in Cycl'Assist, to import roadworks for -instance. All the useful scripts to import OpenData are in the -`scripts/opendata` folder. You can set up a daily cron task to automatically -run the import of roadworks every day. - -#### Updating - -Database migrations are in the `scripts/migrations` folder, labelled by -versions. You should run them in order from your current versions to the -latest one when you upgrade. +Detailed documentation about this software is available under the +[`doc/`](doc/) folder. This covers setting up your own instance, API, privacy +choices etc. ## Contributing Check out the [CONTRIBUTING.md](CONTRIBUTING.md) file for all the required doc -and details before contributing :) Any contributions more than welcome! +and details before contributing :) Any contributions are more than welcome! ## License diff --git a/doc/0.hosting.md b/doc/0.hosting.md new file mode 100644 index 0000000..a2bfaa5 --- /dev/null +++ b/doc/0.hosting.md @@ -0,0 +1,146 @@ +Hosting your own +================ + +The app is made of two separate parts. A (micro-)server part which is +basically just exposing an API on top of the database and a client part +(static JS scripts and assets) which is accessing this API. + +## Updating the app + +Whenever new versions are published, here is a quick guide to do the upgrade: + +* fetch the last updated files from the repository +* ensure the client part build is up to date + (`yarn install && yarn build`) +* check for required migrations (see below) +* ensure the server part requirements are up to date (`pip install -r + requirements.txt`) and restart the server + +From times to times, the database schema might need to be updated. Migrations +(scripts to edit the database schema for you) are provided under the +`scripts/migrations` folder. The scripts in this folder are labelled by +versions, meaning that the `0.3.py` script handles the migration of the +database from the version immediately before `0.3` to the `0.3` version of the +app. + +If you upgrade through several versions at once, you should run all the +migrations scripts for all the intermediate versions, in the ascending order. +There are currently no automated way to handle the updates of the database +schema. + +_Note :_ Versions of the app are listed in the git tags. Current version of +the code is also in the `src/constants.js` file. + + +## Server part + +### Build setup + +``` bash +# Install Python dependencies +pip install -r requirements.txt + +# Start the server +python -m server +``` + +It is better to use a dedicated `virtualenv` if you can, to help manage Python +dependencies in a clean way. + +API routes are all listed within `server/routes.py` file, with documentation +strings. + +### Useful environment variables + +You can pass a few environment variables to the `python -m server` command to +adapt its behavior: + +* `HOST=` to specify the host to listen to (defaults to `127.0.0.1` which + means `localhost` only). +* `PORT=` to specify the port to listen on (defaults to `8081`). +* `DATABASE=` to specify a [database URL](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#db-url) to connect to (defaults to + `sqlite:///reports.db` which means a SQLite database named `reports.db` in + the current working directory). +* `API_TOKEN=` to specify a token required to `POST` data to the API. + +### Serving in production + +You can use the `wsgi.py` script at the root of the git repository to serve +the server side part. You can find some `uwsgi` and `nginx` base config files +under the `support` folder. + +You might also want to put some rate-limiting in front of the API. This can be +done easily when you use `nginx` as a reverse proxy for instance. This is +handled by the `limit_req` directive in the `nginx` base config files provided +in the `support` folder. You can then edit the +`DELAY_BETWEEN_API_BATCH_REQUESTS` configuration option in `src/constants.js` +to ensure that requests will be spaced enough when sending a batch of them so +that they will not be blocked by your rate-limiting. + +### Importing OpenData + +A few OpenData files can be imported in Cycl'Assist, to import roadworks for +instance. All the useful scripts to import OpenData are in the +`scripts/opendata` folder. + +You can set up a daily cron task to automatically run the import of opendata +every day for instance. + + +## Client part + +### Build setup + +Here are the steps to build the client side assets and scripts. + +``` bash +# Install JS dependencies +yarn install + +# Serve with hot reload at localhost:8080 +# (For development only) +yarn dev + +# Build for production with minification +# Output assets and scripts in the `dist/` folder, ready to be used in +# production. +yarn build + +# Build for production and view the bundle analyzer report +# (might be useful for debugging or development) +yarn build --report +``` + +To serve the app in production, you have to build the scripts and assets using +`yarn build`. It will output everything under the `dist/` folder which you can +then serve using any web server (these are just static files to be served). + + +### Useful environment variables + +You can pass a few environment variables to the `yarn build|dev` commands to +adapt the behavior to your needs. + +* `PUBLIC_PATH=https://.../foobar` to serve the app from a subdirectory. +* `API_BASE_URL=https://...` to specify the location of the server API (defaults + to `/`). The value should end with a trailing slash. +* `THUNDERFOREST_API_KEY=` to pass an API key server to use for + [Thunderforest](http://thunderforest.com/) tiles (OpenCycleMap, etc). +* `API_TOKEN=` to pass a token required to access the server side API (check + below in the server part environment variables for more details). + +You should also have a look at the build variables under the `config/` +subdirectory. + + +### Geographical extension + +While the frontend could theoretically work in the entire world without much +modifications, it is currently written with mainland France in mind, mostly +because that is the territory the authors are most familiar with. +Additionnally, this limits the volume of geographical data (such as OSM +extracts) to handle and makes managing the app easier. + +You could of course easily extend it to support other territories. The +French-specific parts of the code so far are: +* The [`AddressInput`](https://framagit.org/phyks/cyclassist/blob/master/src/components/AddressInput.vue) component which uses the [https://adresse.data.gouv.fr/](https://adresse.data.gouv.fr/) API to autocomplete addresses. You could easily replace it with [Algolia Places](https://community.algolia.com/places/) which covers the entire world. diff --git a/doc/10.technical_notes.md b/doc/10.technical_notes.md new file mode 100644 index 0000000..51845f8 --- /dev/null +++ b/doc/10.technical_notes.md @@ -0,0 +1,26 @@ +Technical notes +=============== + +## Privacy design + +The software was built with the idea that your location should only be handled +in the client part. Then, it should never be sent to the server part or any +third party without your knowledge. The app should also provide as much +features as possible without even knowing your location (in case you disabled +geolocation on your device). + +If geolocation is turned off, it falls back to asking you to manually enter a +location through a text field. Of course, without geolocation tracking, some +features no longer makes sense, but you can still browse the reports and +manually add reports at specific known locations. + +If geolocation is turned on, your geolocation data is only handled locally, on +your device. This means that your precise location is never sent to the +server. When fetching reports nearby, all the valid reports from the +server are downloaded and the filtering is done client-side, to avoid sharing +your precise geolocation with the server. + +*Note:* The map tiles (the map background) is downloaded from third +party servers and due to the very nature of the images, it could leak hints +about your geolocation. This might be something addressed in the future by +letting you batch download a given area in advance. diff --git a/doc/20.api.md b/doc/20.api.md new file mode 100644 index 0000000..dab560f --- /dev/null +++ b/doc/20.api.md @@ -0,0 +1,10 @@ +API +=== + +The server part exposes a public API (by default). Read from the API is always +public but writing new reports can be restricted through the use of an +API token (see the doc about deployment for more infos). + +A helper script is available under `scripts/api_doc.py` to export a +documentation of the available API endpoints and usage in the current version +of the code. diff --git a/scripts/api_doc.py b/scripts/api_doc.py new file mode 100755 index 0000000..ac862d6 --- /dev/null +++ b/scripts/api_doc.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +import os +import sys + +import bottle + +SCRIPT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.abspath(os.path.join(SCRIPT_DIRECTORY, '..'))) + +import server.routes + +app_routes = [x for x in bottle.default_app().routes] +for route in app_routes: + if route.method == "OPTIONS": + # Ignore CORS handling + continue + print(route.rule) + print(''.join('=' for _ in route.rule)) + print(route.callback.__doc__) + print('') diff --git a/server/routes.py b/server/routes.py index 44c36d2..2e3bb93 100644 --- a/server/routes.py +++ b/server/routes.py @@ -102,7 +102,28 @@ def get_all_reports(): Example:: - GET /api/v1/reports + > GET /api/v1/reports + + { + "data": [ + { + "attributes": { + "expiration_datetime": null, + "downvotes": 0, + "datetime": "2018-06-27T16:44:12+00:00", + "is_open": true, + "lat": 48.842005, + "upvotes": 1, + "lng": 2.386278, + "type": "interrupt", + … + }, + "type": "reports", + "id": 1 + }, + … + ] + } .. note:: @@ -134,7 +155,28 @@ def get_active_reports(): Example:: - GET /api/v1/reports/active + > GET /api/v1/reports/active + + { + "data": [ + { + "attributes": { + "expiration_datetime": null, + "downvotes": 0, + "datetime": "2018-06-27T16:44:12+00:00", + "is_open": true, + "lat": 48.842005, + "upvotes": 1, + "lng": 2.386278, + "type": "interrupt", + … + }, + "type": "reports", + "id": 1 + }, + … + ] + } .. note:: @@ -165,12 +207,33 @@ def post_report(): Example:: - POST /api/v1/reports + > POST /api/v1/reports + > { + > "type": "pothole", + > "lat": 48.84219652060494, + > "lng": 2.385234797066081 + > } + { - "type": "toto", - "lat": 32, - "lng": 27 + "data": { + "attributes": { + "expiration_datetime": null, + "downvotes": 0, + "datetime": "2018-10-17T13:42:35+00:00", + "first_report_datetime": "2018-10-17T13:42:35+00:00", + "lat": 48.84219652060494, + "upvotes": 0, + "lng": 2.385234797066081, + "type": "pothole", + "is_open": true, + … + }, + "type": "reports", + "id": 1161 + } } + + :return: The newly created report object in a JSON ``data`` dict. """ # Handle CORS if bottle.request.method == 'OPTIONS': @@ -214,7 +277,28 @@ def upvote_report(id): Example:: - POST /api/v1/reports/1/upvote + > POST /api/v1/reports/1/upvote + + { + "data": { + "attributes": { + "expiration_datetime": null, + "downvotes": 0, + "datetime": "2018-10-17T13:42:35+00:00", + "first_report_datetime": "2018-10-17T13:42:35+00:00", + "lat": 48.84219652060494, + "upvotes": 1, + "lng": 2.385234797066081, + "type": "pothole", + "is_open": true, + … + }, + "type": "reports", + "id": 1161 + } + } + + :return: The updated report object in a JSON ``data`` dict. """ # Handle CORS if bottle.request.method == 'OPTIONS': @@ -252,7 +336,28 @@ def downvote_report(id): Example:: - POST /api/v1/reports/1/downvote + > POST /api/v1/reports/1/downvote + + { + "data": { + "attributes": { + "expiration_datetime": null, + "downvotes": 1, + "datetime": "2018-10-17T13:42:35+00:00", + "first_report_datetime": "2018-10-17T13:42:35+00:00", + "lat": 48.84219652060494, + "upvotes": 0, + "lng": 2.385234797066081, + "type": "pothole", + "is_open": true, + … + }, + "type": "reports", + "id": 1161 + } + } + + :return: The updated report object in a JSON ``data`` dict. """ # Handle CORS if bottle.request.method == 'OPTIONS': @@ -282,7 +387,18 @@ def get_stats(): Example:: - GET /api/v1/states + > GET /api/v1/stats + + { + "data": { + "nb_active_reports": 606, + "nb_reports": 1162, + "last_added_report_datetime": "2018-10-17T13:44:16+00:00", + … + } + } + + :return: The available stats about the instance in a JSON ``data`` dict. """ # Handle CORS if bottle.request.method == 'OPTIONS': diff --git a/support/nginx/cyclassist.conf b/support/nginx/cyclassist.conf index 5a51e4f..9fd43e2 100644 --- a/support/nginx/cyclassist.conf +++ b/support/nginx/cyclassist.conf @@ -1,11 +1,13 @@ -# API rate limitation +# Define API rate limitation limit_req_zone $binary_remote_addr zone=cycloAPI:10m rate=1r/s;# UWSGI proxy pass +# Define the server to use upstream, here we assume we serve Cyclassist using +# UWSGI. upstream _cyclassist { server unix:/run/uwsgi/app/cyclassist/socket; } -# Expires map +# Expires map, to ensure correct caching of the assets. map $sent_http_content_type $expires { default off; text/html epoch; @@ -46,7 +48,7 @@ server { # Proxy pass the API calls to the server part location /api { - limit_req zone=cycloAPI burst=3 nodelay; + limit_req zone=cycloAPI burst=3 nodelay; # Add rate-limiting on top of the API include uwsgi_params; uwsgi_pass _cyclassist; }