Rebuild data on import command if required

This commit is contained in:
Lucas Verney 2017-12-06 19:16:24 +01:00
parent 0376cdf4e0
commit c3941bd70c
3 changed files with 9 additions and 15 deletions

View File

@ -158,21 +158,10 @@ def main():
cmds.purge_db(config)
return
# Build data files
try:
force = False
if args.cmd == "build-data":
force = True
if data.preprocess_data(config, force=force):
LOGGER.info("Done building data!")
if args.cmd == "build-data":
sys.exit(0)
except flatisfy.exceptions.DataBuildError as exc:
LOGGER.error("%s", exc)
sys.exit(1)
# Build data files command
if args.cmd == "build-data":
data.preprocess_data(config, force=True)
return
# Fetch command
if args.cmd == "fetch":
# Fetch and filter flats list

View File

@ -168,6 +168,9 @@ def validate_config(config, check_with_data):
assert constraint["postal_codes"]
assert all(isinstance(x, str) for x in constraint["postal_codes"])
if check_with_data:
# Ensure data is built into db
data.preprocess_data(config, force=False)
# Check postal codes
opendata_postal_codes = [
x.postal_code
for x in data.load_data(PostalCode, constraint, config)

View File

@ -60,6 +60,7 @@ def preprocess_data(config, force=False):
session.query(PostalCode).delete()
# Build all opendata files
LOGGER.info("Rebuilding data...")
for preprocess in data_files.PREPROCESSING_FUNCTIONS:
data_objects = preprocess()
if not data_objects:
@ -68,6 +69,7 @@ def preprocess_data(config, force=False):
)
with get_session() as session:
session.add_all(data_objects)
LOGGER.info("Done building data!")
return True