Define new db tables to host constraints

Also make use of advanced types in SQLAlchemy to clean the db scheme.

flatisfy/database/types.py

@@ -1,53 +0,0 @@
-# coding: utf-8
-"""
-This modules implements custom types in SQLAlchemy.
-"""
-from __future__ import absolute_import, print_function, unicode_literals
-
-import json
-
-import sqlalchemy.types as types
-
-
-class StringyJSON(types.TypeDecorator):
-    """
-    Stores and retrieves JSON as TEXT for SQLite.
-
-    From
-    https://avacariu.me/articles/2016/compiling-json-as-text-for-sqlite-with-sqlalchemy.
-
-    .. note ::
-
-        The associated field is immutable. That is, changes to the data
-        (typically, changing the value of a dict field) will not trigger an
-        update on the SQL side upon ``commit`` as the reference to the object
-        will not have been updated. One should force the update by forcing an
-        update of the reference (by performing a ``copy`` operation on the dict
-        for instance).
-    """
-
-    impl = types.TEXT
-
-    def process_bind_param(self, value, dialect):
-        """
-        Process the bound param, serialize the object to JSON before saving
-        into database.
-        """
-        if value is not None:
-            value = json.dumps(value)
-        return value
-
-    def process_result_value(self, value, dialect):
-        """
-        Process the value fetched from the database, deserialize the JSON
-        string before returning the object.
-        """
-        if value is not None:
-            value = json.loads(value)
-        return value
-
-
-# TypeEngine.with_variant says "use StringyJSON instead when
-# connecting to 'sqlite'"
-# pylint: disable=locally-disabled,invalid-name
-MagicJSON = types.JSON().with_variant(StringyJSON, 'sqlite')

flatisfy/models/constraint.py

@@ -0,0 +1,82 @@
+# coding: utf-8
+"""
+This modules defines an SQLAlchemy ORM model for a search constraint.
+"""
+# pylint: disable=locally-disabled,invalid-name,too-few-public-methods
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+from sqlalchemy import (
+    Column, Float, Integer, String
+)
+from sqlalchemy_utils.types.json import JSONType
+from sqlalchemy_utils.types.scalar_list import ScalarListType
+
+import enum
+from sqlalchemy_enum_list import EnumListType
+
+from flatisfy.database.base import BASE
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class HouseTypes(enum.Enum):
+    """
+    An enum of the possible house types.
+    """
+    APART = 0
+    HOUSE = 1
+    PARKING = 2
+    LAND = 3
+    OTHER = 4
+    UNKNOWN = 5
+
+
+class PostTypes(enum.Enum):
+    """
+    An enum of the possible posts types.
+    """
+    RENT = 0
+    SALE = 1
+    SHARING = 2
+
+
+class Constraint(BASE):
+    """
+    SQLAlchemy ORM model to store a search constraint.
+    """
+    __tablename__ = "constraints"
+
+    id = Column(String, primary_key=True)
+    name = Column(String)
+    type = Column(EnumListType(PostTypes, int))
+    house_types = Column(EnumListType(HouseTypes, int))
+    postal_codes = Column(ScalarListType())  # TODO
+    area_min = Column(Float, default=None)  # in m^2
+    area_max = Column(Float, default=None)  # in m^2
+    cost_min = Column(Float, default=None)  # in currency unit
+    cost_max = Column(Float, default=None)  # in currency unit
+    rooms_min = Column(Integer, default=None)
+    rooms_max = Column(Integer, default=None)
+    bedrooms_min = Column(Integer, default=None)
+    bedrooms_max = Column(Integer, default=None)
+    minimum_nb_photos = Column(Integer, default=None)
+    description_should_contain = Column(ScalarListType())  # list of terms
+    time_to = Column(JSONType)  # TODO
+
+    def __repr__(self):
+        return "<Constraint(id=%s, name=%s)>" % (self.id, self.name)
+
+    def json_api_repr(self):
+        """
+        Return a dict representation of this constraint object that is JSON
+        serializable.
+        """
+        constraint_repr = {
+            k: v
+            for k, v in self.__dict__.items()
+            if not k.startswith("_")
+        }
+
+        return constraint_repr

flatisfy/models/flat.py

@@ -11,12 +11,14 @@ import enum
 import arrow
 
 from sqlalchemy import (
-    Column, DateTime, Enum, Float, SmallInteger, String, Text, inspect
+    Column, Enum, Float, SmallInteger, String, Text, inspect
 )
 from sqlalchemy.orm import validates
+from sqlalchemy_utils.types.arrow import ArrowType
+from sqlalchemy_utils.types.json import JSONType
+from sqlalchemy_utils.types.scalar_list import ScalarListType
 
 from flatisfy.database.base import BASE
-from flatisfy.database.types import MagicJSON
 
 
 LOGGER = logging.getLogger(__name__)
@@ -70,32 +72,32 @@ class Flat(BASE):
     cost = Column(Float)
     currency = Column(String)
     utilities = Column(Enum(FlatUtilities), default=FlatUtilities.unknown)
-    date = Column(DateTime)
+    date = Column(ArrowDate)
-    details = Column(MagicJSON)
+    details = Column(JSONType)
     location = Column(String)
     phone = Column(String)
-    photos = Column(MagicJSON)
+    photos = Column(JSONType)
     rooms = Column(Float)
     station = Column(String)
     text = Column(Text)
     title = Column(String)
-    urls = Column(MagicJSON)
+    urls = Column(ScalarListType())
-    merged_ids = Column(MagicJSON)
+    merged_ids = Column(ScalarListType())
     notes = Column(Text)
     notation = Column(SmallInteger, default=0)
 
     # Flatisfy data
     # TODO: Should be in another table with relationships
-    flatisfy_stations = Column(MagicJSON)
+    flatisfy_stations = Column(JSONType)
     flatisfy_postal_code = Column(String)
-    flatisfy_time_to = Column(MagicJSON)
+    flatisfy_time_to = Column(JSONType)
     flatisfy_constraint = Column(String)
 
     # Status
     status = Column(Enum(FlatStatus), default=FlatStatus.new)
 
     # Date for visit
-    visit_date = Column(DateTime)
+    visit_date = Column(ArrowDate)
 
     @validates('utilities')
     def validate_utilities(self, _, utilities):
@@ -138,20 +140,6 @@ class Flat(BASE):
             raise ValueError('notation should be an integer between 0 and 5')
         return notation
 
-    @validates("date")
-    def validate_date(self, _, date):
-        """
-        Date validation method
-        """
-        return arrow.get(date).naive
-
-    @validates("visit_date")
-    def validate_visit_date(self, _, visit_date):
-        """
-        Visit date validation method
-        """
-        return arrow.get(visit_date).naive
-
     @validates("photos")
     def validate_photos(self, _, photos):
         """

requirements.txt

@@ -12,6 +12,8 @@ pillow
 requests
 requests_mock
 sqlalchemy
+sqlalchemy-utils
+SQLAlchemy-Enum-List
 titlecase
 unidecode
 vobject