Browse Source

Rebuild data on import command if required

merge-requests/14/head
Lucas Verney 5 years ago
parent
commit
c3941bd70c
  1. 19
      flatisfy/__main__.py
  2. 3
      flatisfy/config.py
  3. 2
      flatisfy/data.py

19
flatisfy/__main__.py

@ -158,21 +158,10 @@ def main(): @@ -158,21 +158,10 @@ def main():
cmds.purge_db(config)
return
# Build data files
try:
force = False
if args.cmd == "build-data":
force = True
if data.preprocess_data(config, force=force):
LOGGER.info("Done building data!")
if args.cmd == "build-data":
sys.exit(0)
except flatisfy.exceptions.DataBuildError as exc:
LOGGER.error("%s", exc)
sys.exit(1)
# Build data files command
if args.cmd == "build-data":
data.preprocess_data(config, force=True)
return
# Fetch command
if args.cmd == "fetch":
# Fetch and filter flats list

3
flatisfy/config.py

@ -168,6 +168,9 @@ def validate_config(config, check_with_data): @@ -168,6 +168,9 @@ def validate_config(config, check_with_data):
assert constraint["postal_codes"]
assert all(isinstance(x, str) for x in constraint["postal_codes"])
if check_with_data:
# Ensure data is built into db
data.preprocess_data(config, force=False)
# Check postal codes
opendata_postal_codes = [
x.postal_code
for x in data.load_data(PostalCode, constraint, config)

2
flatisfy/data.py

@ -60,6 +60,7 @@ def preprocess_data(config, force=False): @@ -60,6 +60,7 @@ def preprocess_data(config, force=False):
session.query(PostalCode).delete()
# Build all opendata files
LOGGER.info("Rebuilding data...")
for preprocess in data_files.PREPROCESSING_FUNCTIONS:
data_objects = preprocess()
if not data_objects:
@ -68,6 +69,7 @@ def preprocess_data(config, force=False): @@ -68,6 +69,7 @@ def preprocess_data(config, force=False):
)
with get_session() as session:
session.add_all(data_objects)
LOGGER.info("Done building data!")
return True

Loading…
Cancel
Save