{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4500c8d-0e58-4d3f-8dd3-06a4896f397f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import shelve\n",
    "import pickle\n",
    "import zlib\n",
    "import lzma\n",
    "import pytz\n",
    "from datetime import datetime, timedelta\n",
    "from aiomql import MetaTrader, MetaTester, TimeFrame, TestData, AccountInfo, TimeFrame, CopyTicks, Account, GetData\n",
    "# from MetaTrader5 import SymbolInfo\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9307e3df-a5f3-42cf-9d3d-e65e5254c0e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.now()\n",
    "st = now.replace(hour=0, minute=0, second=0, day=1, month=1, year=2023)\n",
    "et = now.replace(hour=9, minute=0, second=0)\n",
    "diff = et - st\n",
    "secs = int(diff.total_seconds())\n",
    "# st = now.replace(hour=0, day=16)\n",
    "# et = now.replace(hour=9)\n",
    "# start = now.replace(day=now.day-3, tzinfo=tz)\n",
    "# end = now.replace(day=now.day-1, tzinfo=tz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63050d62-443a-4d30-b2a6-4ec6d4f00362",
   "metadata": {},
   "outputs": [],
   "source": [
    "symbols = {'Volatility 10 Index', 'Volatility 100 (1s) Index', 'Volatility 25 Index'}\n",
    "timeframes = {TimeFrame.M5, TimeFrame.H1, TimeFrame.M1, TimeFrame.H4, TimeFrame.M30, TimeFrame.M15}\n",
    "gd = GetData(st, et, timeframes, symbols, name='data')\n",
    "await gd.fail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49d86767-d5cc-40a8-894e-c8f5b920f8cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "symbols = {'Volatility 10 Index', 'Volatility 100 (1s) Index', 'Volatility 25 Index'}\n",
    "timeframes = {TimeFrame.M5, TimeFrame.H1, TimeFrame.M1, TimeFrame.H4, TimeFrame.M30, TimeFrame.M15}\n",
    "async with MetaTester(st, et, timeframes, symbols, name='data') as mt:\n",
    "    # res = await mt.copy_ticks_range('Volatility 100 Index', st, et, CopyTicks.ALL)\n",
    "    # print(res)\n",
    "    await mt.get_and_save_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4f8c64b-dcba-40cf-8342-53bb4b3fa6e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "970126ba-a143-4b45-a82e-8cf6b885763c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.set_index(list(range(secs)))\n",
    "df.drop_duplicates(subset=['time'], keep='last', )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf6d4786-4ee1-4bd4-81e2-fa43bdc527a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.set_index('time', drop=False, verify_integrity=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e788c43f-42d3-4c01-9bd9-5224e2175c1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "bg = int(st.timestamp())\n",
    "en = int(secs) + bg\n",
    "index = range(bg, en)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd9cffe6-c550-4f54-be8a-5e55b23b81a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "bf = len(df.index)\n",
    "df = df.reindex(index=index, method='nearest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0b60fb9-0c27-4435-900d-898ea6979e11",
   "metadata": {},
   "outputs": [],
   "source": [
    "last = df.iloc[-1].time\n",
    "print(datetime.fromtimestamp(last, tz=tz), et)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60afd149-9600-4f0e-b3ff-1e8ad10247f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "acc = AccountInfo(balance=500)\n",
    "data = MetaTester(start, end).load_data('data')\n",
    "td = TestData(acc, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be5704d5-4bff-4ec1-8cbe-2e96bfaf4f11",
   "metadata": {},
   "outputs": [],
   "source": [
    "ticks = data['ticks']['Volatility 10 Index']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12b3d5cb-0d31-4b70-b583-1f2becb74a0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "ticks[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee0c4ccb-0e7c-434e-a655-fe89b09e606c",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = TimeFrame.M5\n",
    "y.time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a28ce72-0409-4231-92f7-f4e61fc0be94",
   "metadata": {},
   "outputs": [],
   "source": [
    "end.timestamp()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aeeb64b-d3d2-4053-9808-2539e2755d09",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(ticks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29647ef4-01ad-46a0-afb2-f43a4988a529",
   "metadata": {},
   "outputs": [],
   "source": [
    "type(ticks[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc593a4b-53d6-4203-a986-33b5267b9244",
   "metadata": {},
   "outputs": [],
   "source": [
    "ticks.reshape(8, 86160)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc6440a3-4333-4363-b462-edee4facdd0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a246c6d-d35c-4878-a128-52ed1abb0108",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = np.reshape(ticks, (-1, 8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e9de365-0f13-4d88-8302-f5e3418d489b",
   "metadata": {},
   "outputs": [],
   "source": [
    "ticks.shape = (86160, 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "451b8e09-d17c-4662-a25d-39f00ff01788",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = np.hstack(ticks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "395a1b7b-309d-4e38-b579-1b6b4854b19d",
   "metadata": {},
   "outputs": [],
   "source": [
    "v = np.array((*ticks[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bb16ff4-d92e-4474-a235-d8be87d094d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = next(iter(ticks))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2241c9c-68eb-4371-b3e3-202ebe25f7cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee0d3713-8578-46e2-969c-aebc295fae98",
   "metadata": {},
   "outputs": [],
   "source": [
    "ar = list(range(4))\n",
    "t = np.array(ar)\n",
    "t.shape = (1, 4)\n",
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8502f7da-793e-4bfe-b469-bc109f051507",
   "metadata": {},
   "outputs": [],
   "source": [
    "mt.config.login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d818ddd-f78c-4a50-b4a7-dc266321091c",
   "metadata": {},
   "outputs": [],
   "source": [
    "acc = Account()\n",
    "await acc.sign_in()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c4c32d9-9c45-4ee7-afee-e04d1f5dca4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "mt5 = MetaTrader()\n",
    "sym = await mt5.symbol_info('Volatility 100 (1s) Index')\n",
    "print(sym._asdict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e47bd638-42fa-40d9-b573-474a39884dbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = datetime.now()\n",
    "start = n.replace(hour=0, day=1, year=2020, month=1)\n",
    "end = n.replace(hour=15)\n",
    "res = await mt.copy_rates_range('Volatility 25 Index', TimeFrame.M1, start, end)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4bc7b28-6dac-4863-b257-64d2c5c62575",
   "metadata": {},
   "outputs": [],
   "source": [
    "res = pd.DataFrame(res)\n",
    "len(res.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90c17d01-4ed0-468d-85c4-64988338b8a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a9141a1-7049-48e5-a96f-0b637b817d40",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = shelve.open('./data/01-08-24_17-08-24', writeback=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2d4d4d7-5bfa-43e3-a455-012cf754a106",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = dict(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95931276-fb8c-4a97-b440-330efc5e4d93",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pickle.dumps(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57c620ad-3fa5-41b9-a372-9a090e18ff85",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = zlib.compress(data, level=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d85bcdb-a86a-43f0-9b93-7d8c0ce7cf9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "_data = lzma.compress(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eec34d71-7455-4064-9367-52156407335e",
   "metadata": {},
   "outputs": [],
   "source": [
    "fh = lzma.open('./data/ldata.xz', 'w')\n",
    "fh.write(_data)\n",
    "fh.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f38b857-ed2d-4819-ba7f-4f70dc2fb3f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "rb = lzma.open('./data/ldata.xz')\n",
    "rbb = rb.read()\n",
    "rb.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28220ba7-46af-409b-826d-2979ceb19f65",
   "metadata": {},
   "outputs": [],
   "source": [
    "rbd = lzma.decompress(rbb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3275fd6-70ff-4596-b4ba-9d9e4b6e52fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "rdata = pickle.loads(rbd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bbcbc06-f353-4255-9792-89263793ba0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9830b77-5e73-45c9-bc6a-40c19c3816c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "fh = open('./data/pdata', 'wb')\n",
    "pickle.dump(rdata, fh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "064b8628-1331-4fa3-abf3-66f88c75da33",
   "metadata": {},
   "outputs": [],
   "source": [
    "fh.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe60fc41-8844-4e00-b254-bb95d28528f2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}