From 3145e87a434552fdf93a17a995c0f5bd68e8aa24 Mon Sep 17 00:00:00 2001 From: A Farzat Date: Sat, 11 Oct 2025 09:15:07 +0300 Subject: Organize data_collector to be similar to data_analyser --- data_collector/__main__.py | 10 ++++++++++ data_collector/utils.py | 13 +++++++++++++ 2 files changed, 23 insertions(+) create mode 100755 data_collector/__main__.py create mode 100644 data_collector/utils.py (limited to 'data_collector') diff --git a/data_collector/__main__.py b/data_collector/__main__.py new file mode 100755 index 0000000..11791c2 --- /dev/null +++ b/data_collector/__main__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from time import sleep +from components.database import subscriptions + +from .utils import collect_data + +while True: + collect_data(subscriptions) + sleep(60) diff --git a/data_collector/utils.py b/data_collector/utils.py new file mode 100644 index 0000000..6ca5c69 --- /dev/null +++ b/data_collector/utils.py @@ -0,0 +1,13 @@ +from datetime import datetime, timedelta, UTC + +from pymongo.collection import Collection + +from components.subscriptions.main import Subscription +from components.subscriptions.typing import SubsDict + +def collect_data(subs_collection: Collection[SubsDict]) -> None: + for sub_dict in subs_collection.find(): + sub = Subscription(**sub_dict) + sub._collection = subs_collection + if datetime.now(tz=UTC) - sub.last_fetch > timedelta(seconds=sub.time_between_fetches): + sub.fetch() -- cgit v1.2.3-70-g09d2