Add a CLI argument to import only a given constraint. Drop the necessary constraint.

This commit is contained in:
Lucas Verney 2017-06-19 16:08:06 +02:00
parent b7f9427d53
commit ed7e9dfc1a
5 changed files with 33 additions and 17 deletions

View File

@ -128,10 +128,6 @@ You can think of constraints as "a set of criterias to filter out flats". You
can specify as many constraints as you want, in the configuration file, can specify as many constraints as you want, in the configuration file,
provided that you name each of them uniquely. provided that you name each of them uniquely.
Note that it is mandatory to keep a constraint called `default`, which will be
the default constraint use for refiltering flats already imported with a
constraint that you deleted after the import.
## Building the web assets ## Building the web assets

View File

@ -55,6 +55,10 @@ def parse_args(argv=None):
"-vv", action="store_true", "-vv", action="store_true",
help="Debug logging output." help="Debug logging output."
) )
parent_parser.add_argument(
"--constraints", type=str,
help="Comma-separated list of constraints to consider."
)
# Subcommands # Subcommands
subparsers = parser.add_subparsers( subparsers = parser.add_subparsers(

View File

@ -40,9 +40,15 @@ def filter_flats_list(config, constraint_name, flats_list, fetch_details=True):
try: try:
constraint = config["constraints"][constraint_name] constraint = config["constraints"][constraint_name]
except KeyError: except KeyError:
LOGGER.warning("Missing constraint %s. Using default one.", LOGGER.error(
constraint_name) "Missing constraint %s. Skipping filtering for these posts.",
constraint = config["constraints"]["default"] constraint_name
)
return {
"new": [],
"duplicate": [],
"ignored": []
}
first_pass_result = collections.defaultdict(list) first_pass_result = collections.defaultdict(list)
second_pass_result = collections.defaultdict(list) second_pass_result = collections.defaultdict(list)

View File

@ -97,9 +97,8 @@ def validate_config(config):
# message in the log output. # message in the log output.
# pylint: disable=locally-disabled,line-too-long # pylint: disable=locally-disabled,line-too-long
# Ensure default constraint is here
assert "default" in config["constraints"]
# Ensure constraints are ok # Ensure constraints are ok
assert len(config["constraints"]) > 0
for constraint in config["constraints"].values(): for constraint in config["constraints"].values():
assert "type" in constraint assert "type" in constraint
assert isinstance(constraint["type"], str) assert isinstance(constraint["type"], str)
@ -229,6 +228,18 @@ def load_config(args=None):
"search_index" "search_index"
) )
# Handle constraints filtering
if args and getattr(args, "constraints", None) is not None:
LOGGER.info(
"Filtering constraints from config according to CLI argument."
)
constraints_filter = args.constraints.split(",")
config_data["constraints"] = {
k: v
for k, v in config_data["constraints"].items()
if k in constraints_filter
}
config_validation = validate_config(config_data) config_validation = validate_config(config_data)
if config_validation is True: if config_validation is True:
LOGGER.info("Config has been fully initialized.") LOGGER.info("Config has been fully initialized.")

View File

@ -56,8 +56,7 @@ def flats_v1(config, db):
postal_code_data = next( postal_code_data = next(
x x
for x in postal_codes.get(flat["flatisfy_constraint"], for x in postal_codes.get(flat["flatisfy_constraint"], [])
postal_codes["default"])
if x.postal_code == flat["flatisfy_postal_code"] if x.postal_code == flat["flatisfy_postal_code"]
) )
flat["flatisfy_postal_code"] = { flat["flatisfy_postal_code"] = {
@ -108,10 +107,6 @@ def flat_v1(flat_id, config, db):
""" """
flat = db.query(flat_model.Flat).filter_by(id=flat_id).first() flat = db.query(flat_model.Flat).filter_by(id=flat_id).first()
constraint = config["constraints"].get(flat.flatisfy_constraint,
config["constraints"]["default"])
postal_codes = flatisfy.data.load_data(PostalCode, constraint, config)
if not flat: if not flat:
return bottle.HTTPError(404, "No flat with id {}.".format(flat_id)) return bottle.HTTPError(404, "No flat with id {}.".format(flat_id))
@ -120,6 +115,11 @@ def flat_v1(flat_id, config, db):
try: try:
assert flat["flatisfy_postal_code"] assert flat["flatisfy_postal_code"]
constraint = config["constraints"].get(flat["flatisfy_constraint"],
None)
assert constraint is not None
postal_codes = flatisfy.data.load_data(PostalCode, constraint, config)
postal_code_data = next( postal_code_data = next(
x x
for x in postal_codes for x in postal_codes
@ -276,8 +276,7 @@ def search_v1(db, config):
postal_code_data = next( postal_code_data = next(
x x
for x in postal_codes.get(flat["flatisfy_constraint"], for x in postal_codes.get(flat["flatisfy_constraint"], [])
postal_codes["default"])
if x.postal_code == flat["flatisfy_postal_code"] if x.postal_code == flat["flatisfy_postal_code"]
) )
flat["flatisfy_postal_code"] = { flat["flatisfy_postal_code"] = {