Browse Source

LocalStorage-like for Agents

Implement a localStorage-like API for agents, so that they could store
(limited) data between subsequent calls.

Implement a `FlatFilesStorage` which relies on flat JSON files stored in a tmp dir as an example.

Updated the doc accordingly. Make RATP scenario use this feature.
Phyks (Lucas Verney) 4 years ago
parent
commit
77d68f45b8

+ 1
- 0
CONTRIBUTING.md View File

@@ -22,6 +22,7 @@ Here are some conventions one should respect:
22 22
 
23 23
 - Agents classes always end with `Agent` (e.g. `EchoAgent`).
24 24
 - Meta-agents names always end with `MetaAgent` (e.g. `RATPMetaAgent`).
25
+- Storages names always end with `Storage` (e.g. `FlatFilesStorage`).
25 26
 - The file implementing a given agent (e.g. `SomeAgent`) should be the
26 27
   lowercase agent name (e.g. `someagent.py`).
27 28
 - Some keywords are reserved in Python. Typically, if you have a dict `d`,

+ 14
- 0
doc/00.getting_started.md View File

@@ -140,6 +140,20 @@ python -m infotuyo run_scenario ics_to_caldav config/ics_to_caldav.json
140 140
 ```
141 141
 
142 142
 
143
+## Storages
144
+
145
+Some agents may require a `Storage`. This is a key->value store to store
146
+(limited) number of items between successive runs. This is typically the case
147
+for `TriggerAgent` which needs to store the previous value seen in memory.
148
+
149
+These storages can be implemented through various `Storage` class (see
150
+`infotuyo/storages`). The default one is `FlatFilesStorage` which stores the
151
+items in flat files in the tmp dir.
152
+
153
+To specify which `Storage` should be used in a given scenario, you can use the
154
+`storage_type` scenario metadata to an available `Storage` class name.
155
+
156
+
143 157
 ## Starting the WebUI
144 158
 
145 159
 A WebUI to list scenarios and browse them (very minimalistic for now) is

+ 18
- 1
infotuyo/agents/base.py View File

@@ -8,8 +8,11 @@ having (eventually) some output to pass down the pipeline.
8 8
 """
9 9
 from __future__ import absolute_import, unicode_literals
10 10
 
11
+# System imports
11 12
 import logging
12 13
 
14
+# Local imports
15
+from infotuyo.exceptions import InvalidAgentConfiguration
13 16
 from infotuyo.helpers.jinjadict import JinjaDict
14 17
 
15 18
 
@@ -22,7 +25,11 @@ class Agent(object):
22 25
     # value otherwise.
23 26
     DEFAULT_SETTINGS = {}
24 27
 
25
-    def __init__(self, settings=None, alias=None, comment=None):
28
+    # Whether this agent needs a storage
29
+    HAS_STORAGE = False
30
+
31
+    def __init__(self, settings=None, alias=None, comment=None,
32
+                 storage_builder=None):
26 33
         """
27 34
         Build an agent object given settings and metadata.
28 35
 
@@ -30,7 +37,17 @@ class Agent(object):
30 37
             - settings (dict): A settings dict for this agent.
31 38
             - alias (str): An optional alias for this agent.
32 39
             - comment (str): An optional comment for this agent.
40
+            - storage (str): A function initalizing a storage for this agent.
33 41
         """
42
+        if self.HAS_STORAGE:
43
+            if storage_builder:
44
+                self.storage = storage_builder()
45
+            else:
46
+                raise InvalidAgentConfiguration(
47
+                    "Agent requires a storage but no storage builder provided."
48
+                )
49
+        else:
50
+            self.storage = None
34 51
         self.logger = logging.getLogger(self.__class__.__name__)
35 52
         self.alias = alias
36 53
         self.comment = comment

+ 78
- 0
infotuyo/agents/triggeragent.py View File

@@ -0,0 +1,78 @@
1
+# coding: utf-8
2
+"""
3
+Trigger agent
4
+
5
+An agent which watches a given value and only passes down the input payload
6
+when it changes.
7
+"""
8
+from __future__ import absolute_import, division, unicode_literals
9
+
10
+# Local imports
11
+from infotuyo.agents.base import Agent
12
+from infotuyo.exceptions import InterruptScenarioPropagation
13
+
14
+
15
+class TriggerAgent(Agent):
16
+    """
17
+    An agent to pass down input payload only when the value changed during the
18
+    last two runs.
19
+
20
+    Settings:
21
+        {
22
+            "to_watch": str: "Some expression giving a value to watch."
23
+        }
24
+    Input payload:
25
+        None
26
+    Output payload:
27
+        If there was a change in the watched value: input payload.
28
+        Else: None.
29
+    """
30
+    # Default settings
31
+    DEFAULT_SETTINGS = {
32
+    }
33
+
34
+    # Whether this agent needs a storage
35
+    HAS_STORAGE = True
36
+
37
+    def __init__(self, *args, **kwargs):
38
+        super(TriggerAgent, self).__init__(*args, **kwargs)
39
+
40
+    def run(self, payload_input=None, dry_run=False):
41
+        """
42
+        Run your agent.
43
+
44
+        Params:
45
+            - payload_input (dict): The input payload, which is also the output
46
+                of the previous agent in the pipeline. Defaults to `None` if
47
+                there is no previous agent in the pipeline.
48
+            - dry_run (bool): Whether to run the scenario in dry run mode or
49
+                not.
50
+
51
+        Returns:
52
+            dict: Some payload to pass to the next agent in the pipeline. Can
53
+            be `None`.
54
+        """
55
+        self.logger.info("Start running TriggerAgent.")
56
+        try:
57
+            # Get the old value
58
+            old_value = self.storage.get_item("to_watch")
59
+            self.logger.info("%s",
60
+                             "Old value was: {}.".format(old_value))
61
+            # Only pass down the input if value has changed, interrupt
62
+            # otherwise.
63
+            new_value = self.settings.get(
64
+                "to_watch",
65
+                settings=self.settings,
66
+                payload=payload_input
67
+            )
68
+            if old_value != new_value:
69
+                self.logger.info("New value is different, store new value "
70
+                                 "and continue propagation.")
71
+                self.storage.set_item("to_watch", new_value)
72
+                return payload_input
73
+            else:
74
+                self.logger.info("Value has not changed, interrupt "
75
+                                 "propagation.")
76
+                raise InterruptScenarioPropagation
77
+        finally:
78
+            self.logger.info("Done running TriggerAgent.")

+ 16
- 1
infotuyo/cmd.py View File

@@ -23,6 +23,7 @@ import bottle
23 23
 # Local imports
24 24
 import infotuyo.routes as routes
25 25
 import infotuyo.scenarios.utils
26
+import infotuyo.storage.utils
26 27
 from infotuyo.exceptions import InterruptScenarioPropagation
27 28
 from infotuyo.scenarios.base import Scenario
28 29
 
@@ -42,7 +43,18 @@ def base_argparse_parser():
42 43
         "--scenarios-dir",
43 44
         help=(
44 45
             "An optional folder in which one should look for scenarios "
45
-            "instead of the default path."
46
+            "instead of the default path. You can also use "
47
+            "INFOTUYO_SCENARIOS_DIR env variable."
48
+        ),
49
+        nargs='?',
50
+        default=None
51
+    )
52
+    parser.add_argument(
53
+        "--tmp-dir",
54
+        help=(
55
+            "An optional folder in which infotuyo can store some tmp files "
56
+            "(default to system tmp directory). You can also use "
57
+            "INFOTUYO_TMP_DIR env variable."
46 58
         ),
47 59
         nargs='?',
48 60
         default=None
@@ -62,6 +74,9 @@ def handle_common_args(argv):
62 74
     # Handle scenarios dir
63 75
     if argv.scenarios_dir:
64 76
         infotuyo.scenarios.utils.set_scenarios_dir(argv.scenarios_dir)
77
+    # Handle tmp dir
78
+    if argv.tmp_dir:
79
+        infotuyo.storage.utils.set_tmp_dir(argv.tmp_dir)
65 80
 
66 81
 
67 82
 def serve_cmd(argv):

+ 7
- 0
infotuyo/exceptions.py View File

@@ -13,6 +13,13 @@ class InvalidAgentName(Exception):
13 13
     pass
14 14
 
15 15
 
16
+class InvalidStorageType(Exception):
17
+    """
18
+    Exception representing an invalid storage type.
19
+    """
20
+    pass
21
+
22
+
16 23
 class InvalidScenarioConfiguration(Exception):
17 24
     """
18 25
     Exception raised when the JSON description of a scenario lacks some fields.

+ 13
- 2
infotuyo/scenarios/base.py View File

@@ -14,6 +14,8 @@ import json
14 14
 import sys
15 15
 
16 16
 # Local imports
17
+import infotuyo.storage.utils
18
+
17 19
 from infotuyo.exceptions import InvalidAgentName, InvalidScenarioConfiguration
18 20
 from infotuyo.helpers.jinjadict import JinjaDict
19 21
 
@@ -76,6 +78,9 @@ class Scenario(object):
76 78
         # Load metadata
77 79
         self.name = source["name"]
78 80
         self.description = source.get("description", None)
81
+        self.storage_builder = infotuyo.storage.utils.get_storage_builder(
82
+            source.get("storage_type", None)
83
+        )
79 84
 
80 85
         # Load meta-agents definitions
81 86
         self.meta_agents = {}
@@ -202,13 +207,18 @@ class Scenario(object):
202 207
         )
203 208
 
204 209
         # Create an instance of the agent
210
+        agent_hash = "{}__{}__{}".format(
211
+            self.name, agent_class.__name__, agent_alias if agent_alias else agent_name
212
+        )
213
+        storage_builder = lambda: self.storage_builder(agent_hash)
205 214
         if agent_name in self.meta_agents:
206 215
             # If this is a meta-agent, create an instance of the real
207 216
             # underlying agent
208 217
             instance = agent_class(
209 218
                 agent_settings,
210 219
                 comment=agent_comment,
211
-                alias=agent_alias
220
+                alias=agent_alias,
221
+                storage_builder=storage_builder
212 222
             )
213 223
             # And edit the `run` method to edit the input payload according to
214 224
             # the rules defined in the meta-agent.
@@ -237,7 +247,8 @@ class Scenario(object):
237 247
             instance = agent_class(
238 248
                 agent_settings,
239 249
                 comment=agent_comment,
240
-                alias=agent_alias
250
+                alias=agent_alias,
251
+                storage_builder=storage_builder
241 252
             )
242 253
 
243 254
         # Add the agent to the chain

+ 1
- 0
infotuyo/storage/__init__.py View File

@@ -0,0 +1 @@
1
+# coding: utf-8

+ 41
- 0
infotuyo/storage/base.py View File

@@ -0,0 +1,41 @@
1
+# coding: utf-8
2
+"""
3
+This is the base implementation of a Storage, defining a common interface
4
+across all storage backends available.
5
+
6
+A Storage is comparable to localStorage API on the web. It is a basic API to
7
+store hashable items (everything JSON-serializable) as key->value pairs. It is
8
+not intended to be used to store data on the long term, but rather as a minimal
9
+cache to store single values between agent executions.
10
+
11
+If your Agent needs to actually store lot of data, Storage does not fit and it
12
+should implement its own storage strategy.
13
+"""
14
+
15
+
16
+class Storage(object):
17
+    """
18
+    Base Storage interface (abstract class).
19
+    """
20
+    def __init__(self, agent_hash):
21
+        """
22
+        Build a Storage object given hash. The hash is used to uniquely
23
+        identify the instance of the agent requesting the Storage.
24
+
25
+        Params:
26
+            - agent_hash (str): Some string to uniquely identify this
27
+            particular storage.
28
+        """
29
+        self.hash = agent_hash
30
+
31
+    def get_item(self, key):
32
+        """
33
+        Retrieve the value stored for a given key.
34
+        """
35
+        raise NotImplementedError
36
+
37
+    def set_item(self, key, value):
38
+        """
39
+        Add a key->value matching in the storage.
40
+        """
41
+        raise NotImplementedError

+ 61
- 0
infotuyo/storage/flatfilesstorage.py View File

@@ -0,0 +1,61 @@
1
+# coding: utf-8
2
+"""
3
+This is an implementation of storage based on flat JSON files.
4
+"""
5
+from __future__ import absolute_import, unicode_literals
6
+
7
+import json
8
+import os
9
+
10
+import infotuyo.storage.utils
11
+from infotuyo.storage.base import Storage
12
+
13
+
14
+class FlatFilesStorage(Storage):
15
+    """
16
+    Flat files storage backend.
17
+    """
18
+    def __init__(self, *args, **kwargs):
19
+        """
20
+        Build a Storage object given hash. The hash is used to uniquely
21
+        identify the instance of the agent requesting the Storage.
22
+
23
+        Params:
24
+            - agent_hash (str): Some string to uniquely identify this
25
+            particular storage.
26
+        """
27
+        super(FlatFilesStorage, self).__init__(*args, **kwargs)
28
+
29
+    def _get_file_path(self):
30
+        """
31
+        Get the path to the file for this particular instance.
32
+        """
33
+        return os.path.join(
34
+            infotuyo.storage.utils.get_tmp_dir(),
35
+            "{}.json".format(self.hash)
36
+        )
37
+
38
+    def _get_content(self):
39
+        """
40
+        Get the content of the flat file for this particular instance.
41
+        """
42
+        try:
43
+            with open(self._get_file_path(), 'r') as fh:
44
+                return json.load(fh)
45
+        except IOError:
46
+            return {}
47
+
48
+    def get_item(self, key, default=None):
49
+        """
50
+        Retrieve the value stored for a given key.
51
+        """
52
+        return self._get_content().get(key, default)
53
+
54
+    def set_item(self, key, value):
55
+        """
56
+        Add a key->value matching in the storage.
57
+        """
58
+        content = self._get_content()
59
+        content[key] = value
60
+        with open(self._get_file_path(), 'w') as fh:
61
+            json.dump(content, fh)

+ 74
- 0
infotuyo/storage/utils.py View File

@@ -0,0 +1,74 @@
1
+# coding: utf-8
2
+"""
3
+Storage management
4
+"""
5
+from __future__ import absolute_import, unicode_literals
6
+
7
+# System imports
8
+import importlib
9
+import os
10
+import tempfile
11
+
12
+# Local imports
13
+from infotuyo.exceptions import InvalidStorageType
14
+from infotuyo.storage.flatfilesstorage import FlatFilesStorage
15
+
16
+
17
+def get_storage_builder(storage_type=None):
18
+    """
19
+    Get the storage builder given the storage type as str.
20
+
21
+    Params:
22
+        - storage_type (str): Type of storage to instantiate. (Optional,
23
+            default storage type is FlatFilesStorage)
24
+
25
+    Returns:
26
+        A function (or class __init__) taking an agent hash as argument and
27
+        returning an instance of a Storage.
28
+    """
29
+    # Default storage builder
30
+    storage_builder = FlatFilesStorage
31
+
32
+    if storage_type is not None:
33
+        try:
34
+            module = importlib.import_module(
35
+                "infotuyo.storage.{}".format(storage_type.lower())
36
+            )
37
+            storage_builder = getattr(module, storage_type)
38
+        except (AttributeError, ImportError):
39
+            raise InvalidStorageType(
40
+                "Storage {} does not exist.".format(storage_type)
41
+            )
42
+
43
+    return storage_builder
44
+
45
+
46
+def get_tmp_dir():
47
+    """
48
+    Get the tmp dir to use to write output files.
49
+
50
+    Returns:
51
+        str: The full path.
52
+    """
53
+    tmp_dir = os.environ.get(
54
+        "INFOTUYO_TMP_DIR",
55
+        None
56
+    )
57
+    if tmp_dir is None:
58
+        tmp_dir = os.path.join(
59
+            tempfile.gettempdir(),
60
+            "infotuyo"
61
+        )
62
+        if not os.path.exists(tmp_dir):
63
+            os.makedirs(tmp_dir)
64
+    return tmp_dir
65
+
66
+
67
+def set_tmp_dir(tmp_dir):
68
+    """
69
+    Set the environment variable indicating the tmp dir to use.
70
+
71
+    Args:
72
+        tmp_dir (str): Path to the tmp dir.
73
+    """
74
+    os.environ["INFOTUYO_TMP_DIR"] = tmp_dir

+ 6
- 0
scenarios/ratp_notifier.json View File

@@ -46,6 +46,12 @@
46 46
                 "label": ""
47 47
             }
48 48
         },
49
+        {
50
+            "name": "TriggerAgent",
51
+            "settings": {
52
+                "to_watch": "{{ payload.msg }}"
53
+            }
54
+        },
49 55
         {
50 56
             "name": "ConditionAgent",
51 57
             "comment": "Do not send text message if no message to send.",