Stash

Also make use of advanced types in SQLAlchemy to clean the db scheme.

flatisfy/database/__init__.py

@@ -13,7 +13,12 @@ from sqlalchemy.engine import Engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.exc import OperationalError, SQLAlchemyError
 
+# Import models
+import flatisfy.models.postal_code  # noqa: F401
+import flatisfy.models.public_transport  # noqa: F401
 import flatisfy.models.flat  # noqa: F401
+import flatisfy.models.constraint  # noqa: F401
+
 from flatisfy.database.base import BASE
 from flatisfy.database.whooshalchemy import IndexService
 

flatisfy/database/types.py

@@ -1,53 +0,0 @@
-# coding: utf-8
-"""
-This modules implements custom types in SQLAlchemy.
-"""
-from __future__ import absolute_import, print_function, unicode_literals
-
-import json
-
-import sqlalchemy.types as types
-
-
-class StringyJSON(types.TypeDecorator):
-    """
-    Stores and retrieves JSON as TEXT for SQLite.
-
-    From
-    https://avacariu.me/articles/2016/compiling-json-as-text-for-sqlite-with-sqlalchemy.
-
-    .. note ::
-
-        The associated field is immutable. That is, changes to the data
-        (typically, changing the value of a dict field) will not trigger an
-        update on the SQL side upon ``commit`` as the reference to the object
-        will not have been updated. One should force the update by forcing an
-        update of the reference (by performing a ``copy`` operation on the dict
-        for instance).
-    """
-
-    impl = types.TEXT
-
-    def process_bind_param(self, value, dialect):
-        """
-        Process the bound param, serialize the object to JSON before saving
-        into database.
-        """
-        if value is not None:
-            value = json.dumps(value)
-        return value
-
-    def process_result_value(self, value, dialect):
-        """
-        Process the value fetched from the database, deserialize the JSON
-        string before returning the object.
-        """
-        if value is not None:
-            value = json.loads(value)
-        return value
-
-
-# TypeEngine.with_variant says "use StringyJSON instead when
-# connecting to 'sqlite'"
-# pylint: disable=locally-disabled,invalid-name
-MagicJSON = types.JSON().with_variant(StringyJSON, 'sqlite')

flatisfy/models/constraint.py

@@ -0,0 +1,100 @@
+# coding: utf-8
+"""
+This modules defines an SQLAlchemy ORM model for a search constraint.
+"""
+# pylint: disable=locally-disabled,invalid-name,too-few-public-methods
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+from sqlalchemy import (
+    Column, Float, ForeignKey, Integer, String, Table
+)
+from sqlalchemy.orm import relationship
+from sqlalchemy_utils.types.json import JSONType
+from sqlalchemy_utils.types.scalar_list import ScalarListType
+
+import enum
+from sqlalchemy_enum_list import EnumListType
+
+from flatisfy.database.base import BASE
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class HouseTypes(enum.Enum):
+    """
+    An enum of the possible house types.
+    """
+    APART = 0
+    HOUSE = 1
+    PARKING = 2
+    LAND = 3
+    OTHER = 4
+    UNKNOWN = 5
+
+
+class PostTypes(enum.Enum):
+    """
+    An enum of the possible posts types.
+    """
+    RENT = 0
+    SALE = 1
+    SHARING = 2
+
+
+association_table = Table(
+    'constraint_postal_codes_association', BASE.metadata,
+    Column('constraint_id', Integer, ForeignKey('constraints.id')),
+    Column('postal_code_id', Integer, ForeignKey('postal_codes.id'))
+)
+
+
+class Constraint(BASE):
+    """
+    SQLAlchemy ORM model to store a search constraint.
+    """
+    __tablename__ = "constraints"
+
+    id = Column(String, primary_key=True)
+    name = Column(String)
+    type = Column(EnumListType(PostTypes, int))
+    house_types = Column(EnumListType(HouseTypes, int))
+    # TODO: What happens when one delete a postal code?
+    postal_codes = relationship("PostalCode", secondary=association_table)
+
+    area_min = Column(Float, default=None)  # in m^2
+    area_max = Column(Float, default=None)  # in m^2
+
+    cost_min = Column(Float, default=None)  # in currency unit
+    cost_max = Column(Float, default=None)  # in currency unit
+
+    rooms_min = Column(Integer, default=None)
+    rooms_max = Column(Integer, default=None)
+
+    bedrooms_min = Column(Integer, default=None)
+    bedrooms_max = Column(Integer, default=None)
+
+    minimum_nb_photos = Column(Integer, default=None)
+    description_should_contain = Column(ScalarListType())  # list of terms
+
+    # Dict mapping names to {"gps": [lat, lng], "time": (min, max) }
+    # ``min`` and ``max`` are in seconds and can be ``null``.
+    # TODO: Use an additional time_to_places table?
+    time_to = Column(JSONType)
+
+    def __repr__(self):
+        return "<Constraint(id=%s, name=%s)>" % (self.id, self.name)
+
+    def json_api_repr(self):
+        """
+        Return a dict representation of this constraint object that is JSON
+        serializable.
+        """
+        constraint_repr = {
+            k: v
+            for k, v in self.__dict__.items()
+            if not k.startswith("_")
+        }
+
+        return constraint_repr

flatisfy/models/flat.py

@@ -8,15 +8,16 @@ from __future__ import absolute_import, print_function, unicode_literals
 import logging
 import enum
 
-import arrow
-
 from sqlalchemy import (
-    Column, DateTime, Enum, Float, SmallInteger, String, Text, inspect
+    Column, Enum, Float, ForeignKey, Integer, SmallInteger, String, Table,
+    Text, inspect
 )
-from sqlalchemy.orm import validates
+from sqlalchemy.orm import relationship, validates
+from sqlalchemy_utils.types.arrow import ArrowType
+from sqlalchemy_utils.types.json import JSONType
+from sqlalchemy_utils.types.scalar_list import ScalarListType
 
 from flatisfy.database.base import BASE
-from flatisfy.database.types import MagicJSON
 
 
 LOGGER = logging.getLogger(__name__)
@@ -53,6 +54,14 @@ AUTOMATED_STATUSES = [
     FlatStatus.ignored
 ]
 
+stations_association_table = Table(
+    'stations_flats_association', BASE.metadata,
+    Column(
+        'public_transport_id', Integer, ForeignKey('public_transports.id')
+    ),
+    Column('flat_id', Integer, ForeignKey('flats.id'))
+)
+
 
 class Flat(BASE):
     """
@@ -70,32 +79,41 @@ class Flat(BASE):
     cost = Column(Float)
     currency = Column(String)
     utilities = Column(Enum(FlatUtilities), default=FlatUtilities.unknown)
-    date = Column(DateTime)
-    details = Column(MagicJSON)
+    date = Column(ArrowType)
+    details = Column(JSONType)
     location = Column(String)
     phone = Column(String)
-    photos = Column(MagicJSON)
+    photos = Column(JSONType)
     rooms = Column(Float)
     station = Column(String)
     text = Column(Text)
     title = Column(String)
-    urls = Column(MagicJSON)
-    merged_ids = Column(MagicJSON)
+    urls = Column(ScalarListType())
+    merged_ids = Column(ScalarListType())
     notes = Column(Text)
     notation = Column(SmallInteger, default=0)
 
-    # Flatisfy data
-    # TODO: Should be in another table with relationships
-    flatisfy_stations = Column(MagicJSON)
-    flatisfy_postal_code = Column(String)
-    flatisfy_time_to = Column(MagicJSON)
-    flatisfy_constraint = Column(String)
+    # Flatisfy found stations
+    # TODO: What happens when one deletes a station?
+    flatisfy_stations = relationship("PublicTransport",
+                                     secondary=stations_association_table)
+    # Flatisfy found postal code
+    # TODO: What happens when one deletes a postal code?
+    flatisfy_postal_code_id = Column(Integer, ForeignKey('postal_codes.id'))
+    flatisfy_postal_code = relationship("PostalCode")
+    # Computed time to
+    flatisfy_time_to = Column(JSONType)
+    # Constraint relationship
+    # TODO: What happens when one deletes a constraint?
+    # TODO: A flat could match multiple constraints
+    flatisfy_constraint_id = Column(Integer, ForeignKey('constraints.id'))
+    flatisfy_constraint = relationship("Constraint")
 
     # Status
     status = Column(Enum(FlatStatus), default=FlatStatus.new)
 
     # Date for visit
-    visit_date = Column(DateTime)
+    visit_date = Column(ArrowType)
 
     @validates('utilities')
     def validate_utilities(self, _, utilities):
@@ -138,20 +156,6 @@ class Flat(BASE):
             raise ValueError('notation should be an integer between 0 and 5')
         return notation
 
-    @validates("date")
-    def validate_date(self, _, date):
-        """
-        Date validation method
-        """
-        return arrow.get(date).naive
-
-    @validates("visit_date")
-    def validate_visit_date(self, _, visit_date):
-        """
-        Visit date validation method
-        """
-        return arrow.get(visit_date).naive
-
     @validates("photos")
     def validate_photos(self, _, photos):
         """

requirements.txt

@@ -12,6 +12,8 @@ pillow
 requests
 requests_mock
 sqlalchemy
+sqlalchemy-utils
+SQLAlchemy-Enum-List
 titlecase
 unidecode
 vobject