From ed7e9dfc1a9090a0a06b3efc4b33ba58b4a13299 Mon Sep 17 00:00:00 2001 From: "Phyks (Lucas Verney)" Date: Mon, 19 Jun 2017 16:08:06 +0200 Subject: [PATCH] Add a CLI argument to import only a given constraint. Drop the necessary constraint. --- doc/0.getting_started.md | 4 ---- flatisfy/__main__.py | 4 ++++ flatisfy/cmds.py | 12 +++++++++--- flatisfy/config.py | 15 +++++++++++++-- flatisfy/web/routes/api.py | 15 +++++++-------- 5 files changed, 33 insertions(+), 17 deletions(-) diff --git a/doc/0.getting_started.md b/doc/0.getting_started.md index 84c083f..cd97ea4 100644 --- a/doc/0.getting_started.md +++ b/doc/0.getting_started.md @@ -128,10 +128,6 @@ You can think of constraints as "a set of criterias to filter out flats". You can specify as many constraints as you want, in the configuration file, provided that you name each of them uniquely. -Note that it is mandatory to keep a constraint called `default`, which will be -the default constraint use for refiltering flats already imported with a -constraint that you deleted after the import. - ## Building the web assets diff --git a/flatisfy/__main__.py b/flatisfy/__main__.py index 119fb3d..a266c76 100644 --- a/flatisfy/__main__.py +++ b/flatisfy/__main__.py @@ -55,6 +55,10 @@ def parse_args(argv=None): "-vv", action="store_true", help="Debug logging output." ) + parent_parser.add_argument( + "--constraints", type=str, + help="Comma-separated list of constraints to consider." + ) # Subcommands subparsers = parser.add_subparsers( diff --git a/flatisfy/cmds.py b/flatisfy/cmds.py index a0f5371..7683a58 100644 --- a/flatisfy/cmds.py +++ b/flatisfy/cmds.py @@ -40,9 +40,15 @@ def filter_flats_list(config, constraint_name, flats_list, fetch_details=True): try: constraint = config["constraints"][constraint_name] except KeyError: - LOGGER.warning("Missing constraint %s. Using default one.", - constraint_name) - constraint = config["constraints"]["default"] + LOGGER.error( + "Missing constraint %s. Skipping filtering for these posts.", + constraint_name + ) + return { + "new": [], + "duplicate": [], + "ignored": [] + } first_pass_result = collections.defaultdict(list) second_pass_result = collections.defaultdict(list) diff --git a/flatisfy/config.py b/flatisfy/config.py index 443d958..11e09a2 100644 --- a/flatisfy/config.py +++ b/flatisfy/config.py @@ -97,9 +97,8 @@ def validate_config(config): # message in the log output. # pylint: disable=locally-disabled,line-too-long - # Ensure default constraint is here - assert "default" in config["constraints"] # Ensure constraints are ok + assert len(config["constraints"]) > 0 for constraint in config["constraints"].values(): assert "type" in constraint assert isinstance(constraint["type"], str) @@ -229,6 +228,18 @@ def load_config(args=None): "search_index" ) + # Handle constraints filtering + if args and getattr(args, "constraints", None) is not None: + LOGGER.info( + "Filtering constraints from config according to CLI argument." + ) + constraints_filter = args.constraints.split(",") + config_data["constraints"] = { + k: v + for k, v in config_data["constraints"].items() + if k in constraints_filter + } + config_validation = validate_config(config_data) if config_validation is True: LOGGER.info("Config has been fully initialized.") diff --git a/flatisfy/web/routes/api.py b/flatisfy/web/routes/api.py index 1894bf0..3628b30 100644 --- a/flatisfy/web/routes/api.py +++ b/flatisfy/web/routes/api.py @@ -56,8 +56,7 @@ def flats_v1(config, db): postal_code_data = next( x - for x in postal_codes.get(flat["flatisfy_constraint"], - postal_codes["default"]) + for x in postal_codes.get(flat["flatisfy_constraint"], []) if x.postal_code == flat["flatisfy_postal_code"] ) flat["flatisfy_postal_code"] = { @@ -108,10 +107,6 @@ def flat_v1(flat_id, config, db): """ flat = db.query(flat_model.Flat).filter_by(id=flat_id).first() - constraint = config["constraints"].get(flat.flatisfy_constraint, - config["constraints"]["default"]) - postal_codes = flatisfy.data.load_data(PostalCode, constraint, config) - if not flat: return bottle.HTTPError(404, "No flat with id {}.".format(flat_id)) @@ -120,6 +115,11 @@ def flat_v1(flat_id, config, db): try: assert flat["flatisfy_postal_code"] + constraint = config["constraints"].get(flat["flatisfy_constraint"], + None) + assert constraint is not None + postal_codes = flatisfy.data.load_data(PostalCode, constraint, config) + postal_code_data = next( x for x in postal_codes @@ -276,8 +276,7 @@ def search_v1(db, config): postal_code_data = next( x - for x in postal_codes.get(flat["flatisfy_constraint"], - postal_codes["default"]) + for x in postal_codes.get(flat["flatisfy_constraint"], []) if x.postal_code == flat["flatisfy_postal_code"] ) flat["flatisfy_postal_code"] = {