RAPTOR v18.4: Исправлена отчетность, активированы выходные
This commit is contained in:
root
@@ -0,0 +1,70 @@
|
||||
import logging
|
||||
from dataclasses import replace
|
||||
from typing import Dict, Generic, Tuple, TypeVar, cast
|
||||
|
||||
from t_tech.invest import InstrumentIdType
|
||||
from t_tech.invest.caching.instruments_cache.models import (
|
||||
InstrumentResponse,
|
||||
InstrumentsResponse,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
TInstrumentResponse = TypeVar("TInstrumentResponse", bound=InstrumentResponse)
|
||||
TInstrumentsResponse = TypeVar("TInstrumentsResponse", bound=InstrumentsResponse)
|
||||
|
||||
|
||||
class InstrumentStorage(Generic[TInstrumentResponse, TInstrumentsResponse]):
|
||||
def __init__(self, instruments_response: TInstrumentsResponse):
|
||||
self._instruments_response = instruments_response
|
||||
|
||||
self._instrument_by_class_code_figi: Dict[
|
||||
Tuple[str, str], InstrumentResponse
|
||||
] = {
|
||||
(instrument.class_code, instrument.figi): instrument
|
||||
for instrument in self._instruments_response.instruments
|
||||
}
|
||||
self._instrument_by_class_code_ticker: Dict[
|
||||
Tuple[str, str], InstrumentResponse
|
||||
] = {
|
||||
(instrument.class_code, instrument.ticker): instrument
|
||||
for instrument in self._instruments_response.instruments
|
||||
}
|
||||
self._instrument_by_class_code_uid: Dict[
|
||||
Tuple[str, str], InstrumentResponse
|
||||
] = {
|
||||
(instrument.class_code, instrument.uid): instrument
|
||||
for instrument in self._instruments_response.instruments
|
||||
}
|
||||
|
||||
# fmt: off
|
||||
self._instrument_by_class_code_id_index = {
|
||||
InstrumentIdType.INSTRUMENT_ID_UNSPECIFIED:
|
||||
self._instrument_by_class_code_figi,
|
||||
InstrumentIdType.INSTRUMENT_ID_TYPE_FIGI:
|
||||
self._instrument_by_class_code_figi,
|
||||
InstrumentIdType.INSTRUMENT_ID_TYPE_TICKER:
|
||||
self._instrument_by_class_code_ticker,
|
||||
InstrumentIdType.INSTRUMENT_ID_TYPE_UID:
|
||||
self._instrument_by_class_code_uid,
|
||||
}
|
||||
# fmt: on
|
||||
|
||||
def get(
|
||||
self, *, id_type: InstrumentIdType, class_code: str, id: str
|
||||
) -> TInstrumentResponse:
|
||||
logger.debug(
|
||||
"Cache request id_type=%s, class_code=%s, id=%s", id_type, class_code, id
|
||||
)
|
||||
instrument_by_class_code_id = self._instrument_by_class_code_id_index[id_type]
|
||||
logger.debug(
|
||||
"Index for %s found: \n%s", id_type, instrument_by_class_code_id.keys()
|
||||
)
|
||||
key = (class_code, id)
|
||||
logger.debug("Cache request key=%s", key)
|
||||
|
||||
return cast(TInstrumentResponse, instrument_by_class_code_id[key])
|
||||
|
||||
def get_instruments_response(self) -> TInstrumentsResponse:
|
||||
return replace(self._instruments_response, **{})
|
||||
@@ -0,0 +1,203 @@
|
||||
import logging
|
||||
from typing import cast
|
||||
|
||||
from t_tech.invest import (
|
||||
Bond,
|
||||
BondResponse,
|
||||
BondsResponse,
|
||||
CurrenciesResponse,
|
||||
Currency,
|
||||
CurrencyResponse,
|
||||
Etf,
|
||||
EtfResponse,
|
||||
EtfsResponse,
|
||||
Future,
|
||||
FutureResponse,
|
||||
FuturesResponse,
|
||||
InstrumentIdType,
|
||||
InstrumentStatus,
|
||||
Share,
|
||||
ShareResponse,
|
||||
SharesResponse,
|
||||
)
|
||||
from t_tech.invest.caching.instruments_cache.instrument_storage import InstrumentStorage
|
||||
from t_tech.invest.caching.instruments_cache.interface import IInstrumentsGetter
|
||||
from t_tech.invest.caching.instruments_cache.models import (
|
||||
InstrumentResponse,
|
||||
InstrumentsResponse,
|
||||
)
|
||||
from t_tech.invest.caching.instruments_cache.protocol import InstrumentsResponseCallable
|
||||
from t_tech.invest.caching.instruments_cache.settings import InstrumentsCacheSettings
|
||||
from t_tech.invest.caching.overrides import TTLCache
|
||||
from t_tech.invest.services import InstrumentsService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InstrumentsCache(IInstrumentsGetter):
|
||||
def __init__(
|
||||
self,
|
||||
settings: InstrumentsCacheSettings,
|
||||
instruments_service: InstrumentsService,
|
||||
):
|
||||
self._settings = settings
|
||||
self._instruments_service = instruments_service
|
||||
|
||||
logger.debug("Initialising instruments cache")
|
||||
self._instruments_methods = [
|
||||
self.shares,
|
||||
self.futures,
|
||||
self.etfs,
|
||||
self.bonds,
|
||||
self.currencies,
|
||||
]
|
||||
self._cache: TTLCache = TTLCache(
|
||||
maxsize=len(self._instruments_methods),
|
||||
ttl=self._settings.ttl.total_seconds(),
|
||||
)
|
||||
self._refresh_cache()
|
||||
|
||||
def _refresh_cache(self):
|
||||
logger.debug("Refreshing instruments cache")
|
||||
for instruments_method in self._instruments_methods:
|
||||
instruments_method()
|
||||
self._assert_cache()
|
||||
|
||||
def _assert_cache(self):
|
||||
if self._cache.keys() != {f.__name__ for f in self._instruments_methods}:
|
||||
raise KeyError(f"Cache does not have all instrument types {self._cache}")
|
||||
|
||||
def _get_instrument_storage(
|
||||
self, get_instruments_method: InstrumentsResponseCallable
|
||||
) -> InstrumentStorage[InstrumentResponse, InstrumentsResponse]:
|
||||
storage_key = get_instruments_method.__name__
|
||||
storage = self._cache.get(storage_key)
|
||||
if storage is not None:
|
||||
logger.debug("Got storage for key %s from cache", storage_key)
|
||||
return storage
|
||||
logger.debug(
|
||||
"Storage for key %s not found, creating new storage with ttl=%s",
|
||||
storage_key,
|
||||
self._cache.ttl,
|
||||
)
|
||||
instruments_response = get_instruments_method(
|
||||
instrument_status=InstrumentStatus.INSTRUMENT_STATUS_ALL
|
||||
)
|
||||
storage = InstrumentStorage(instruments_response=instruments_response)
|
||||
self._cache[storage_key] = storage
|
||||
return storage # noqa:R504
|
||||
|
||||
def shares(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> SharesResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[ShareResponse, SharesResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.shares),
|
||||
)
|
||||
return storage.get_instruments_response()
|
||||
|
||||
def share_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> ShareResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[Share, SharesResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.shares),
|
||||
)
|
||||
share = storage.get(id_type=id_type, class_code=class_code, id=id)
|
||||
return ShareResponse(instrument=share)
|
||||
|
||||
def futures(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> FuturesResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[FutureResponse, FuturesResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.futures),
|
||||
)
|
||||
return storage.get_instruments_response()
|
||||
|
||||
def future_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> FutureResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[Future, FuturesResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.futures),
|
||||
)
|
||||
future = storage.get(id_type=id_type, class_code=class_code, id=id)
|
||||
return FutureResponse(instrument=future)
|
||||
|
||||
def etfs(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> EtfsResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[EtfResponse, EtfsResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.etfs),
|
||||
)
|
||||
return storage.get_instruments_response()
|
||||
|
||||
def etf_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> EtfResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[Etf, EtfsResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.etfs),
|
||||
)
|
||||
etf = storage.get(id_type=id_type, class_code=class_code, id=id)
|
||||
return EtfResponse(instrument=etf)
|
||||
|
||||
def bonds(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> BondsResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[BondResponse, BondsResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.bonds),
|
||||
)
|
||||
return storage.get_instruments_response()
|
||||
|
||||
def bond_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> BondResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[Bond, BondsResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.bonds),
|
||||
)
|
||||
bond = storage.get(id_type=id_type, class_code=class_code, id=id)
|
||||
return BondResponse(instrument=bond)
|
||||
|
||||
def currencies(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> CurrenciesResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[CurrencyResponse, CurrenciesResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.currencies),
|
||||
)
|
||||
return storage.get_instruments_response()
|
||||
|
||||
def currency_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> CurrencyResponse:
|
||||
storage = cast(
|
||||
InstrumentStorage[Currency, CurrenciesResponse], # type: ignore
|
||||
self._get_instrument_storage(self._instruments_service.currencies),
|
||||
)
|
||||
currency = storage.get(id_type=id_type, class_code=class_code, id=id)
|
||||
return CurrencyResponse(instrument=currency)
|
||||
@@ -0,0 +1,98 @@
|
||||
import abc
|
||||
|
||||
from t_tech.invest import (
|
||||
BondResponse,
|
||||
BondsResponse,
|
||||
CurrenciesResponse,
|
||||
CurrencyResponse,
|
||||
EtfResponse,
|
||||
EtfsResponse,
|
||||
FutureResponse,
|
||||
FuturesResponse,
|
||||
InstrumentIdType,
|
||||
InstrumentStatus,
|
||||
ShareResponse,
|
||||
SharesResponse,
|
||||
)
|
||||
|
||||
|
||||
class IInstrumentsGetter(abc.ABC):
|
||||
@abc.abstractmethod
|
||||
def shares(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> SharesResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def share_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> ShareResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def futures(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> FuturesResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def future_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> FutureResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def etfs(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> EtfsResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def etf_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> EtfResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def bonds(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> BondsResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def bond_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> BondResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def currencies(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> CurrenciesResponse:
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def currency_by(
|
||||
self,
|
||||
*,
|
||||
id_type: InstrumentIdType = InstrumentIdType(0),
|
||||
class_code: str = "",
|
||||
id: str = "",
|
||||
) -> CurrencyResponse:
|
||||
pass
|
||||
@@ -0,0 +1,13 @@
|
||||
from typing import List
|
||||
|
||||
|
||||
class InstrumentResponse:
|
||||
class_code: str
|
||||
|
||||
figi: str
|
||||
ticker: str
|
||||
uid: str
|
||||
|
||||
|
||||
class InstrumentsResponse:
|
||||
instruments: List[InstrumentResponse]
|
||||
@@ -0,0 +1,14 @@
|
||||
from typing import Protocol
|
||||
|
||||
from t_tech.invest import InstrumentStatus
|
||||
from t_tech.invest.caching.instruments_cache.models import InstrumentsResponse
|
||||
|
||||
|
||||
class InstrumentsResponseCallable(Protocol):
|
||||
def __call__(
|
||||
self, *, instrument_status: InstrumentStatus = InstrumentStatus(0)
|
||||
) -> InstrumentsResponse:
|
||||
...
|
||||
|
||||
def __name__(self) -> str:
|
||||
...
|
||||
@@ -0,0 +1,7 @@
|
||||
import dataclasses
|
||||
from datetime import timedelta
|
||||
|
||||
|
||||
@dataclasses.dataclass()
|
||||
class InstrumentsCacheSettings:
|
||||
ttl: timedelta = timedelta(days=1)
|
||||
@@ -0,0 +1,156 @@
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Generator, Iterable, Optional, Tuple
|
||||
|
||||
from t_tech.invest import CandleInterval, HistoricCandle
|
||||
from t_tech.invest.caching.market_data_cache.cache_settings import (
|
||||
MarketDataCacheSettings,
|
||||
)
|
||||
from t_tech.invest.caching.market_data_cache.datetime_range import DatetimeRange
|
||||
from t_tech.invest.caching.market_data_cache.instrument_date_range_market_data import (
|
||||
InstrumentDateRangeData,
|
||||
)
|
||||
from t_tech.invest.caching.market_data_cache.instrument_market_data_storage import (
|
||||
InstrumentMarketDataStorage,
|
||||
)
|
||||
from t_tech.invest.schemas import CandleSource
|
||||
from t_tech.invest.services import Services
|
||||
from t_tech.invest.utils import (
|
||||
candle_interval_to_timedelta,
|
||||
floor_datetime,
|
||||
now,
|
||||
round_datetime_range,
|
||||
with_filtering_distinct_candles,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MarketDataCache:
|
||||
def __init__(self, settings: MarketDataCacheSettings, services: Services):
|
||||
self._settings = settings
|
||||
self._settings.base_cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._services = services
|
||||
self._figi_cache_storages: Dict[
|
||||
Tuple[str, CandleInterval], InstrumentMarketDataStorage
|
||||
] = {}
|
||||
|
||||
def _get_candles_from_net(
|
||||
self,
|
||||
figi: str,
|
||||
interval: CandleInterval,
|
||||
from_: datetime,
|
||||
to: datetime,
|
||||
instrument_id: str = "",
|
||||
candle_source_type: Optional[CandleSource] = None,
|
||||
) -> Iterable[HistoricCandle]:
|
||||
yield from self._services.get_all_candles(
|
||||
figi=figi,
|
||||
interval=interval,
|
||||
from_=from_,
|
||||
to=to,
|
||||
instrument_id=instrument_id,
|
||||
candle_source_type=candle_source_type,
|
||||
)
|
||||
|
||||
def _with_saving_into_cache(
|
||||
self,
|
||||
storage: InstrumentMarketDataStorage,
|
||||
from_net: Iterable[HistoricCandle],
|
||||
) -> Iterable[HistoricCandle]:
|
||||
candles = list(from_net)
|
||||
if candles:
|
||||
complete_candles = list(self._filter_complete_candles(candles))
|
||||
complete_candle_times = [candle.time for candle in complete_candles]
|
||||
complete_net_range = (
|
||||
min(complete_candle_times),
|
||||
max(complete_candle_times),
|
||||
)
|
||||
storage.update(
|
||||
[
|
||||
InstrumentDateRangeData(
|
||||
date_range=complete_net_range,
|
||||
historic_candles=complete_candles,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
yield from candles
|
||||
|
||||
def _filter_complete_candles(
|
||||
self, candles: Iterable[HistoricCandle]
|
||||
) -> Iterable[HistoricCandle]:
|
||||
return filter(lambda candle: candle.is_complete, candles)
|
||||
|
||||
@with_filtering_distinct_candles # type: ignore
|
||||
def get_all_candles(
|
||||
self,
|
||||
*,
|
||||
from_: datetime,
|
||||
to: Optional[datetime] = None,
|
||||
interval: CandleInterval = CandleInterval(0),
|
||||
figi: str = "",
|
||||
instrument_id: str = "",
|
||||
candle_source_type: Optional[CandleSource] = None,
|
||||
) -> Generator[HistoricCandle, None, None]:
|
||||
interval_delta = candle_interval_to_timedelta(interval)
|
||||
to = to or now()
|
||||
|
||||
processed_time = from_
|
||||
figi_cache_storage = self._get_figi_cache_storage(figi=figi, interval=interval)
|
||||
for cached in figi_cache_storage.get(
|
||||
request_range=round_datetime_range(
|
||||
date_range=(from_, to), interval=interval
|
||||
)
|
||||
):
|
||||
cached_start, cached_end = cached.date_range
|
||||
cached_candles = list(cached.historic_candles)
|
||||
if cached_start > processed_time:
|
||||
yield from self._with_saving_into_cache(
|
||||
storage=figi_cache_storage,
|
||||
from_net=self._get_candles_from_net(
|
||||
figi=figi,
|
||||
interval=interval,
|
||||
from_=processed_time,
|
||||
to=cached_start,
|
||||
instrument_id=instrument_id,
|
||||
candle_source_type=candle_source_type,
|
||||
),
|
||||
)
|
||||
|
||||
yield from cached_candles
|
||||
processed_time = cached_end
|
||||
|
||||
if processed_time + interval_delta <= to:
|
||||
yield from self._with_saving_into_cache(
|
||||
storage=figi_cache_storage,
|
||||
from_net=self._get_candles_from_net(
|
||||
figi=figi,
|
||||
interval=interval,
|
||||
from_=processed_time,
|
||||
to=to,
|
||||
instrument_id=instrument_id,
|
||||
),
|
||||
)
|
||||
|
||||
figi_cache_storage.merge()
|
||||
|
||||
def _get_figi_cache_storage(
|
||||
self, figi: str, interval: CandleInterval
|
||||
) -> InstrumentMarketDataStorage:
|
||||
figi_tuple = (figi, interval)
|
||||
storage = self._figi_cache_storages.get(figi_tuple)
|
||||
if storage is None:
|
||||
storage = InstrumentMarketDataStorage(
|
||||
figi=figi, interval=interval, settings=self._settings
|
||||
)
|
||||
self._figi_cache_storages[figi_tuple] = storage
|
||||
return storage # noqa:R504
|
||||
|
||||
def _round_net_range(
|
||||
self, net_range: DatetimeRange, interval_delta: timedelta
|
||||
) -> DatetimeRange:
|
||||
start, end = net_range
|
||||
return floor_datetime(start, interval_delta), floor_datetime(
|
||||
end, interval_delta
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import enum
|
||||
import logging
|
||||
import os
|
||||
import pickle # noqa:S403 # nosec
|
||||
from pathlib import Path
|
||||
from typing import Dict, Generator, Sequence
|
||||
|
||||
from t_tech.invest.caching.market_data_cache.datetime_range import DatetimeRange
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MarketDataCacheFormat(str, enum.Enum):
|
||||
CSV = "csv"
|
||||
|
||||
|
||||
@dataclasses.dataclass()
|
||||
class MarketDataCacheSettings:
|
||||
base_cache_dir: Path = Path(os.getcwd()) / ".market_data_cache"
|
||||
format_extension: MarketDataCacheFormat = MarketDataCacheFormat.CSV
|
||||
field_names: Sequence[str] = (
|
||||
"time",
|
||||
"open",
|
||||
"high",
|
||||
"low",
|
||||
"close",
|
||||
"volume",
|
||||
"is_complete",
|
||||
"candle_source",
|
||||
"volume_buy",
|
||||
"volume_sell",
|
||||
)
|
||||
meta_extension: str = "meta"
|
||||
|
||||
|
||||
@dataclasses.dataclass()
|
||||
class FileMetaData:
|
||||
cached_range_in_file: Dict[DatetimeRange, Path]
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def meta_file_context(meta_file_path: Path) -> Generator[FileMetaData, None, None]:
|
||||
try:
|
||||
with open(meta_file_path, "rb") as f:
|
||||
meta = pickle.load(f) # noqa:S301 # nosec
|
||||
except FileNotFoundError:
|
||||
logger.error("File %s was not found. Creating default.", meta_file_path)
|
||||
|
||||
meta = FileMetaData(cached_range_in_file={})
|
||||
try:
|
||||
yield meta
|
||||
finally:
|
||||
with open(meta_file_path, "wb") as f:
|
||||
pickle.dump(meta, f)
|
||||
@@ -0,0 +1,4 @@
|
||||
from datetime import datetime
|
||||
from typing import Tuple
|
||||
|
||||
DatetimeRange = Tuple[datetime, datetime]
|
||||
@@ -0,0 +1,11 @@
|
||||
import dataclasses
|
||||
from typing import Iterable
|
||||
|
||||
from t_tech.invest.caching.market_data_cache.datetime_range import DatetimeRange
|
||||
from t_tech.invest.schemas import HistoricCandle
|
||||
|
||||
|
||||
@dataclasses.dataclass()
|
||||
class InstrumentDateRangeData:
|
||||
date_range: DatetimeRange
|
||||
historic_candles: Iterable[HistoricCandle]
|
||||
@@ -0,0 +1,291 @@
|
||||
import csv
|
||||
import dataclasses
|
||||
import itertools
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Generator, Iterable, Iterator, Optional, Tuple
|
||||
|
||||
import dateutil.parser
|
||||
|
||||
from t_tech.invest.caching.market_data_cache.cache_settings import (
|
||||
MarketDataCacheSettings,
|
||||
meta_file_context,
|
||||
)
|
||||
from t_tech.invest.caching.market_data_cache.datetime_range import DatetimeRange
|
||||
from t_tech.invest.caching.market_data_cache.instrument_date_range_market_data import (
|
||||
InstrumentDateRangeData,
|
||||
)
|
||||
from t_tech.invest.caching.market_data_cache.interface import (
|
||||
IInstrumentMarketDataStorage,
|
||||
)
|
||||
from t_tech.invest.caching.market_data_cache.serialization import custom_asdict_factory
|
||||
from t_tech.invest.schemas import CandleInterval, HistoricCandle
|
||||
from t_tech.invest.utils import dataclass_from_dict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class InstrumentMarketDataStorage(
|
||||
IInstrumentMarketDataStorage[Iterable[InstrumentDateRangeData]]
|
||||
):
|
||||
def __init__(
|
||||
self, figi: str, interval: CandleInterval, settings: MarketDataCacheSettings
|
||||
):
|
||||
self._figi = figi
|
||||
self._interval = interval
|
||||
self._settings = settings
|
||||
self._settings.base_cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._meta_path = self._get_metafile(
|
||||
file=self._get_base_file_path(figi=self._figi, interval=self._interval)
|
||||
)
|
||||
|
||||
def _get_base_file_path(self, figi: str, interval: CandleInterval) -> Path:
|
||||
instrument_dir = self._get_cache_dir_for_instrument(figi=figi)
|
||||
instrument_dir.mkdir(parents=True, exist_ok=True)
|
||||
return self._get_cache_file_for_instrument(
|
||||
instrument_dir=instrument_dir, interval=interval
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _datetime_to_safe_filename(dt: datetime) -> str:
|
||||
return str(int(dt.timestamp()))
|
||||
|
||||
def _get_file_path(self, date_range: DatetimeRange) -> Path:
|
||||
start, end = date_range
|
||||
filepath = self._get_base_file_path(figi=self._figi, interval=self._interval)
|
||||
start_str = self._datetime_to_safe_filename(start)
|
||||
end_str = self._datetime_to_safe_filename(end)
|
||||
filepath = filepath.parent / (filepath.name + f"-{start_str}-{end_str}")
|
||||
return filepath.with_suffix(f".{self._settings.format_extension.value}")
|
||||
|
||||
def _get_metafile(self, file: Path) -> Path:
|
||||
return file.with_suffix(f".{self._settings.meta_extension}")
|
||||
|
||||
def _get_cache_file_for_instrument(
|
||||
self, instrument_dir: Path, interval: CandleInterval
|
||||
) -> Path:
|
||||
return instrument_dir / interval.name
|
||||
|
||||
def _get_cache_dir_for_instrument(self, figi: str) -> Path:
|
||||
return self._settings.base_cache_dir / figi
|
||||
|
||||
def _get_range_from_file(
|
||||
self, reader: Iterable[Dict], request_range: DatetimeRange
|
||||
) -> Iterable[Dict]:
|
||||
start, end = request_range
|
||||
for row in reader:
|
||||
row_time = dateutil.parser.parse(row["time"])
|
||||
if start <= row_time <= end:
|
||||
yield row
|
||||
if end < row_time:
|
||||
return
|
||||
|
||||
def _get_candles_from_cache(
|
||||
self,
|
||||
file: Path,
|
||||
request_range: DatetimeRange,
|
||||
) -> Generator[HistoricCandle, None, None]:
|
||||
with open(file, "r") as infile: # pylint: disable=W1514
|
||||
reader = csv.DictReader(infile, fieldnames=self._settings.field_names)
|
||||
reader_iter = iter(reader)
|
||||
next(reader_iter) # pylint: disable=R1708
|
||||
for row in self._get_range_from_file(
|
||||
reader_iter, request_range=request_range
|
||||
):
|
||||
yield self._candle_from_row(row)
|
||||
|
||||
def _order_rows(
|
||||
self, dict_reader1: Iterator[Dict], dict_reader2: Iterator[Dict]
|
||||
) -> Iterable[Dict]:
|
||||
dict_reader_iter1 = iter(dict_reader1)
|
||||
dict_reader_iter2 = iter(dict_reader2)
|
||||
|
||||
while True:
|
||||
try:
|
||||
candle_dict1 = next(dict_reader_iter1)
|
||||
except StopIteration:
|
||||
yield from dict_reader_iter2
|
||||
break
|
||||
try:
|
||||
candle_dict2 = next(dict_reader_iter2)
|
||||
except StopIteration:
|
||||
yield from dict_reader_iter1
|
||||
break
|
||||
|
||||
candle_dict_time1 = dateutil.parser.parse(candle_dict1["time"])
|
||||
candle_dict_time2 = dateutil.parser.parse(candle_dict2["time"])
|
||||
if candle_dict_time1 > candle_dict_time2:
|
||||
dict_reader_iter1 = itertools.chain([candle_dict1], dict_reader_iter1)
|
||||
yield candle_dict2
|
||||
elif candle_dict_time1 < candle_dict_time2:
|
||||
dict_reader_iter2 = itertools.chain([candle_dict2], dict_reader_iter2)
|
||||
yield candle_dict1
|
||||
else:
|
||||
yield candle_dict1
|
||||
|
||||
def _order_candles(
|
||||
self,
|
||||
tmp_dict_reader: Iterator[Dict],
|
||||
historic_candles: Iterable[HistoricCandle],
|
||||
) -> Iterable[Dict]:
|
||||
tmp_iter = iter(tmp_dict_reader)
|
||||
candle_iter = iter(historic_candles)
|
||||
|
||||
while True:
|
||||
try:
|
||||
tmp_candle_dict = next(tmp_iter)
|
||||
except StopIteration:
|
||||
yield from [dataclasses.asdict(candle) for candle in candle_iter]
|
||||
break
|
||||
try:
|
||||
candle = next(candle_iter)
|
||||
except StopIteration:
|
||||
yield from tmp_iter
|
||||
break
|
||||
|
||||
tmp_candle_time = dateutil.parser.parse(tmp_candle_dict["time"])
|
||||
if tmp_candle_time > candle.time:
|
||||
tmp_iter = itertools.chain([tmp_candle_dict], tmp_iter)
|
||||
yield dataclasses.asdict(candle)
|
||||
elif tmp_candle_time < candle.time:
|
||||
candle_iter = itertools.chain([candle], candle_iter)
|
||||
yield tmp_candle_dict
|
||||
else:
|
||||
yield tmp_candle_dict
|
||||
|
||||
def _write_candles_file(self, data: InstrumentDateRangeData) -> Path:
|
||||
file = self._get_file_path(date_range=data.date_range)
|
||||
with open(file, mode="w") as csv_file: # pylint: disable=W1514
|
||||
writer = csv.DictWriter(csv_file, fieldnames=self._settings.field_names)
|
||||
writer.writeheader()
|
||||
for candle in data.historic_candles:
|
||||
writer.writerow(
|
||||
dataclasses.asdict(candle, dict_factory=custom_asdict_factory)
|
||||
)
|
||||
return file
|
||||
|
||||
def _candle_from_row(self, row: Dict[str, str]) -> HistoricCandle:
|
||||
return dataclass_from_dict(HistoricCandle, row)
|
||||
|
||||
def _get_intersection(
|
||||
self,
|
||||
request_range: DatetimeRange,
|
||||
cached_range: DatetimeRange,
|
||||
) -> Optional[DatetimeRange]:
|
||||
request_start, request_end = request_range
|
||||
cached_start, cached_end = cached_range
|
||||
max_start = max(request_start, cached_start)
|
||||
min_end = min(request_end, cached_end)
|
||||
if max_start <= min_end:
|
||||
return max_start, min_end
|
||||
return None
|
||||
|
||||
def _merge_intersecting_files( # pylint: disable=R0914
|
||||
self,
|
||||
file1: Path,
|
||||
range1: DatetimeRange,
|
||||
file2: Path,
|
||||
range2: DatetimeRange,
|
||||
) -> Tuple[DatetimeRange, Path]:
|
||||
new_range = (min(min(range1), min(range2)), max(max(range1), max(range2)))
|
||||
new_file = self._get_file_path(date_range=new_range)
|
||||
|
||||
if new_file == file1:
|
||||
file2.unlink()
|
||||
return new_range, new_file
|
||||
if new_file == file2:
|
||||
file1.unlink()
|
||||
return new_range, new_file
|
||||
|
||||
with open(file1, "r") as infile1: # pylint: disable=W1514
|
||||
reader1 = csv.DictReader(infile1, fieldnames=self._settings.field_names)
|
||||
reader_iter1 = iter(reader1)
|
||||
next(reader_iter1) # skip header
|
||||
|
||||
with open(file2, "r") as infile2: # pylint: disable=W1514
|
||||
reader2 = csv.DictReader(infile2, fieldnames=self._settings.field_names)
|
||||
reader_iter2 = iter(reader2)
|
||||
next(reader_iter2) # skip header
|
||||
|
||||
with open(new_file, mode="w") as csv_file: # pylint: disable=W1514
|
||||
writer = csv.DictWriter(
|
||||
csv_file, fieldnames=self._settings.field_names
|
||||
)
|
||||
writer.writeheader()
|
||||
|
||||
for candle_dict in self._order_rows(
|
||||
dict_reader1=reader_iter1, dict_reader2=reader_iter2
|
||||
):
|
||||
writer.writerow(candle_dict)
|
||||
|
||||
file1.unlink()
|
||||
file2.unlink()
|
||||
return new_range, new_file
|
||||
|
||||
def _get_distinct_product(
|
||||
self, cached_range_in_file: Dict[DatetimeRange, Path]
|
||||
) -> Iterable[Tuple[Tuple[DatetimeRange, Path], Tuple[DatetimeRange, Path]]]:
|
||||
sorted_items = self._get_cached_items_sorted_by_start(cached_range_in_file)
|
||||
for i, items1 in enumerate(sorted_items): # pylint: disable=R1702
|
||||
for j, items2 in enumerate(sorted_items):
|
||||
if i < j:
|
||||
yield items1, items2
|
||||
|
||||
def _try_merge_files(
|
||||
self, cached_range_in_file: Dict[DatetimeRange, Path]
|
||||
) -> Dict[DatetimeRange, Path]:
|
||||
new_cached_range_in_file = cached_range_in_file.copy()
|
||||
file_pairs = self._get_distinct_product(new_cached_range_in_file)
|
||||
for (cached_range, cached_file), (cached_range2, cached_file2) in file_pairs:
|
||||
intersection_range = self._get_intersection(
|
||||
request_range=cached_range2, cached_range=cached_range
|
||||
)
|
||||
if intersection_range is not None:
|
||||
merged_range, merged_file = self._merge_intersecting_files(
|
||||
file1=cached_file,
|
||||
range1=cached_range,
|
||||
file2=cached_file2,
|
||||
range2=cached_range2,
|
||||
)
|
||||
del new_cached_range_in_file[cached_range]
|
||||
del new_cached_range_in_file[cached_range2]
|
||||
new_cached_range_in_file[merged_range] = merged_file
|
||||
return self._try_merge_files(new_cached_range_in_file)
|
||||
return new_cached_range_in_file
|
||||
|
||||
def get(self, request_range: DatetimeRange) -> Iterable[InstrumentDateRangeData]:
|
||||
with meta_file_context(meta_file_path=self._meta_path) as meta_file:
|
||||
cached_range_in_file = meta_file.cached_range_in_file
|
||||
|
||||
for cached_range, cached_file in self._get_cached_items_sorted_by_start(
|
||||
cached_range_in_file
|
||||
):
|
||||
intersection = self._get_intersection(
|
||||
request_range=request_range, cached_range=cached_range
|
||||
)
|
||||
if intersection is not None:
|
||||
candles = self._get_candles_from_cache(
|
||||
cached_file, request_range=request_range
|
||||
)
|
||||
yield InstrumentDateRangeData(
|
||||
date_range=intersection, historic_candles=candles
|
||||
)
|
||||
|
||||
def update(self, data_list: Iterable[InstrumentDateRangeData]):
|
||||
with meta_file_context(meta_file_path=self._meta_path) as meta_file:
|
||||
for data in data_list:
|
||||
new_file = self._write_candles_file(data)
|
||||
meta_file.cached_range_in_file[data.date_range] = new_file
|
||||
|
||||
def merge(self):
|
||||
with meta_file_context(meta_file_path=self._meta_path) as meta_file:
|
||||
new_cached_range_in_file = self._try_merge_files(
|
||||
meta_file.cached_range_in_file
|
||||
)
|
||||
meta_file.cached_range_in_file = new_cached_range_in_file
|
||||
|
||||
def _get_cached_items_sorted_by_start(
|
||||
self, cached_range_in_file: Dict[DatetimeRange, Path]
|
||||
) -> Iterable[Tuple[DatetimeRange, Path]]:
|
||||
return sorted(cached_range_in_file.items(), key=lambda pair: pair[0][0])
|
||||
@@ -0,0 +1,13 @@
|
||||
from typing import Protocol, TypeVar
|
||||
|
||||
from t_tech.invest.caching.market_data_cache.datetime_range import DatetimeRange
|
||||
|
||||
TInstrumentData = TypeVar("TInstrumentData")
|
||||
|
||||
|
||||
class IInstrumentMarketDataStorage(Protocol[TInstrumentData]):
|
||||
def get(self, request_range: DatetimeRange) -> TInstrumentData:
|
||||
pass
|
||||
|
||||
def update(self, data_list: TInstrumentData):
|
||||
pass
|
||||
@@ -0,0 +1,10 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
def custom_asdict_factory(data):
|
||||
def convert_value(obj):
|
||||
if isinstance(obj, Enum):
|
||||
return obj.value
|
||||
return obj
|
||||
|
||||
return {k: convert_value(v) for k, v in data}
|
||||
10
invest-python-master/t_tech/invest/caching/overrides.py
Normal file
10
invest-python-master/t_tech/invest/caching/overrides.py
Normal file
@@ -0,0 +1,10 @@
|
||||
import time
|
||||
|
||||
from cachetools import TTLCache as TTLCacheBase
|
||||
|
||||
|
||||
class TTLCache(TTLCacheBase):
|
||||
def __init__(self, maxsize, ttl, timer=None, getsizeof=None):
|
||||
if timer is None:
|
||||
timer = time.monotonic
|
||||
super().__init__(maxsize=maxsize, ttl=ttl, timer=timer, getsizeof=getsizeof)
|
||||
Reference in New Issue
Block a user