From 3406cc1ca161780ec7d0cff944cbeaf6441309fd Mon Sep 17 00:00:00 2001 From: Ram Date: Sun, 16 Feb 2020 20:58:12 +0100 Subject: [PATCH] initial commit --- .editorconfig | 21 + .github/ISSUE_TEMPLATE.md | 15 + .gitignore | 105 +++ .travis.yml | 29 + AUTHORS.rst | 13 + CONTRIBUTING.rst | 128 ++++ HISTORY.rst | 8 + LICENSE | 22 + MANIFEST.in | 11 + Makefile | 85 +++ README.rst | 37 ++ docs/Makefile | 20 + docs/authors.rst | 1 + docs/conf.py | 162 +++++ docs/contributing.rst | 1 + docs/history.rst | 1 + docs/index.rst | 20 + docs/installation.rst | 51 ++ docs/make.bat | 36 ++ docs/readme.rst | 1 + docs/usage.rst | 7 + mql5_zmq_backtrader/__init__.py | 11 + mql5_zmq_backtrader/adapter.py | 34 + mql5_zmq_backtrader/cli.py | 16 + mql5_zmq_backtrader/mql5_zmq_backtrader.py | 1 + mql5_zmq_backtrader/mt5broker.py | 344 ++++++++++ mql5_zmq_backtrader/mt5data.py | 244 +++++++ mql5_zmq_backtrader/mt5store.py | 715 +++++++++++++++++++++ requirements_dev.txt | 10 + setup.cfg | 22 + setup.py | 53 ++ tests/__init__.py | 1 + tests/test_mql5_zmq_backtrader.py | 33 + tox.ini | 20 + 34 files changed, 2278 insertions(+) create mode 100644 .editorconfig create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 AUTHORS.rst create mode 100644 CONTRIBUTING.rst create mode 100644 HISTORY.rst create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 README.rst create mode 100644 docs/Makefile create mode 100644 docs/authors.rst create mode 100755 docs/conf.py create mode 100644 docs/contributing.rst create mode 100644 docs/history.rst create mode 100644 docs/index.rst create mode 100644 docs/installation.rst create mode 100644 docs/make.bat create mode 100644 docs/readme.rst create mode 100644 docs/usage.rst create mode 100644 mql5_zmq_backtrader/__init__.py create mode 100755 mql5_zmq_backtrader/adapter.py create mode 100644 mql5_zmq_backtrader/cli.py create mode 100644 mql5_zmq_backtrader/mql5_zmq_backtrader.py create mode 100755 mql5_zmq_backtrader/mt5broker.py create mode 100755 mql5_zmq_backtrader/mt5data.py create mode 100755 mql5_zmq_backtrader/mt5store.py create mode 100644 requirements_dev.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/test_mql5_zmq_backtrader.py create mode 100644 tox.ini diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d4a2c44 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# http://editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 +end_of_line = lf + +[*.bat] +indent_style = tab +end_of_line = crlf + +[LICENSE] +insert_final_newline = false + +[Makefile] +indent_style = tab diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..a2f58f8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,15 @@ +* mql5_zmq_backtrader version: +* Python version: +* Operating System: + +### Description + +Describe what you were trying to get done. +Tell us what happened, what went wrong, and what you expected to happen. + +### What I Did + +``` +Paste the command(s) you ran and the output. +If there was a crash, please include the traceback here. +``` diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..43091aa --- /dev/null +++ b/.gitignore @@ -0,0 +1,105 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# IDE settings +.vscode/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..89a07fe --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +# Config file for automatic testing at travis-ci.com + +language: python +python: + - 3.8 + - 3.7 + - 3.6 + - 3.5 + +# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors +install: pip install -U tox-travis + +# Command to run tests, e.g. python setup.py test +script: tox + +# Assuming you have installed the travis-ci CLI tool, after you +# create the Github repo and add it to Travis, run the +# following command to finish PyPI deployment setup: +# $ travis encrypt --add deploy.password +deploy: + provider: pypi + distributions: sdist bdist_wheel + user: parrondo + password: + secure: PLEASE_REPLACE_ME + on: + tags: true + repo: parrondo/mql5_zmq_backtrader + python: 3.8 diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..62474c1 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,13 @@ +======= +Credits +======= + +Development Lead +---------------- + +* R. Martin Parrondo + +Contributors +------------ + +None yet. Why not be the first? diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..3e72233 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,128 @@ +.. highlight:: shell + +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/parrondo/mql5_zmq_backtrader/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +mql5_zmq_backtrader could always use more documentation, whether as part of the +official mql5_zmq_backtrader docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/parrondo/mql5_zmq_backtrader/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `mql5_zmq_backtrader` for local development. + +1. Fork the `mql5_zmq_backtrader` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:your_name_here/mql5_zmq_backtrader.git + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ mkvirtualenv mql5_zmq_backtrader + $ cd mql5_zmq_backtrader/ + $ python setup.py develop + +4. Create a branch for local development:: + + $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8 and the + tests, including testing other Python versions with tox:: + + $ flake8 mql5_zmq_backtrader tests + $ python setup.py test or pytest + $ tox + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub:: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.5, 3.6, 3.7 and 3.8, and for PyPy. Check + https://travis-ci.com/parrondo/mql5_zmq_backtrader/pull_requests + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + + + $ python -m unittest tests.test_mql5_zmq_backtrader + +Deploying +--------- + +A reminder for the maintainers on how to deploy. +Make sure all your changes are committed (including an entry in HISTORY.rst). +Then run:: + +$ bump2version patch # possible: major / minor / patch +$ git push +$ git push --tags + +Travis will then deploy to PyPI if tests pass. diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..4abb1c3 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,8 @@ +======= +History +======= + +0.1.0 (2020-02-16) +------------------ + +* First release on PyPI. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f99e424 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2020, R. Martin Parrondo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..965b2dd --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.rst +include LICENSE +include README.rst + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a3d80d5 --- /dev/null +++ b/Makefile @@ -0,0 +1,85 @@ +.PHONY: clean clean-test clean-pyc clean-build docs help +.DEFAULT_GOAL := help + +define BROWSER_PYSCRIPT +import os, webbrowser, sys + +from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +BROWSER := python -c "$$BROWSER_PYSCRIPT" + +help: + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-build: ## remove build artifacts + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: ## remove Python file artifacts + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + rm -fr .pytest_cache + +lint: ## check style with flake8 + flake8 mql5_zmq_backtrader tests + +test: ## run tests quickly with the default Python + python setup.py test + +test-all: ## run tests on every Python version with tox + tox + +coverage: ## check code coverage quickly with the default Python + coverage run --source mql5_zmq_backtrader setup.py test + coverage report -m + coverage html + $(BROWSER) htmlcov/index.html + +docs: ## generate Sphinx HTML documentation, including API docs + rm -f docs/mql5_zmq_backtrader.rst + rm -f docs/modules.rst + sphinx-apidoc -o docs/ mql5_zmq_backtrader + $(MAKE) -C docs clean + $(MAKE) -C docs html + $(BROWSER) docs/_build/html/index.html + +servedocs: docs ## compile the docs watching for changes + watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + +release: dist ## package and upload a release + twine upload dist/* + +dist: clean ## builds source and wheel package + python setup.py sdist + python setup.py bdist_wheel + ls -l dist + +install: clean ## install the package to the active Python's site-packages + python setup.py install diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..e1ddf14 --- /dev/null +++ b/README.rst @@ -0,0 +1,37 @@ +=================== +mql5_zmq_backtrader +=================== + + +.. image:: https://img.shields.io/pypi/v/mql5_zmq_backtrader.svg + :target: https://pypi.python.org/pypi/mql5_zmq_backtrader + +.. image:: https://img.shields.io/travis/parrondo/mql5_zmq_backtrader.svg + :target: https://travis-ci.com/parrondo/mql5_zmq_backtrader + +.. image:: https://readthedocs.org/projects/mql5-zmq-backtrader/badge/?version=latest + :target: https://mql5-zmq-backtrader.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + + + + +Project developed to work as a server for Python trading community. It is based on ZeroMQ sockets and uses JSON format to communicate messages. It is a python library for the ZeroMQ API within backtrader framework. It allows rapid trading algo development. For details of API behavior, please see the online API document. + + +* Free software: MIT license +* Documentation: https://mql5-zmq-backtrader.readthedocs.io. + + +Features +-------- + +* TODO + +Credits +------- + +This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..7bb78f0 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = mql5_zmq_backtrader +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/authors.rst b/docs/authors.rst new file mode 100644 index 0000000..e122f91 --- /dev/null +++ b/docs/authors.rst @@ -0,0 +1 @@ +.. include:: ../AUTHORS.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100755 index 0000000..94cb7e8 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# +# mql5_zmq_backtrader documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 9 13:47:02 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another +# directory, add these directories to sys.path here. If the directory is +# relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +import mql5_zmq_backtrader + +# -- General configuration --------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'mql5_zmq_backtrader' +copyright = "2020, R. Martin Parrondo" +author = "R. Martin Parrondo" + +# The version info for the project you're documenting, acts as replacement +# for |version| and |release|, also used in various other places throughout +# the built documents. +# +# The short X.Y version. +version = mql5_zmq_backtrader.__version__ +# The full version, including alpha/beta/rc tags. +release = mql5_zmq_backtrader.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a +# theme further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output --------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'mql5_zmq_backtraderdoc' + + +# -- Options for LaTeX output ------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'mql5_zmq_backtrader.tex', + 'mql5_zmq_backtrader Documentation', + 'R. Martin Parrondo', 'manual'), +] + + +# -- Options for manual page output ------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'mql5_zmq_backtrader', + 'mql5_zmq_backtrader Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'mql5_zmq_backtrader', + 'mql5_zmq_backtrader Documentation', + author, + 'mql5_zmq_backtrader', + 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..e582053 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..2506499 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1 @@ +.. include:: ../HISTORY.rst diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..f3423b0 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +Welcome to mql5_zmq_backtrader's documentation! +====================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + readme + installation + usage + modules + contributing + authors + history + +Indices and tables +================== +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..270abb2 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,51 @@ +.. highlight:: shell + +============ +Installation +============ + + +Stable release +-------------- + +To install mql5_zmq_backtrader, run this command in your terminal: + +.. code-block:: console + + $ pip install mql5_zmq_backtrader + +This is the preferred method to install mql5_zmq_backtrader, as it will always install the most recent stable release. + +If you don't have `pip`_ installed, this `Python installation guide`_ can guide +you through the process. + +.. _pip: https://pip.pypa.io +.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ + + +From sources +------------ + +The sources for mql5_zmq_backtrader can be downloaded from the `Github repo`_. + +You can either clone the public repository: + +.. code-block:: console + + $ git clone git://github.com/parrondo/mql5_zmq_backtrader + +Or download the `tarball`_: + +.. code-block:: console + + $ curl -OJL https://github.com/parrondo/mql5_zmq_backtrader/tarball/master + +Once you have a copy of the source, you can install it with: + +.. code-block:: console + + $ python setup.py install + + +.. _Github repo: https://github.com/parrondo/mql5_zmq_backtrader +.. _tarball: https://github.com/parrondo/mql5_zmq_backtrader/tarball/master diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..1ac699f --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=python -msphinx +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=mql5_zmq_backtrader + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..72a3355 --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..7c4377f --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,7 @@ +===== +Usage +===== + +To use mql5_zmq_backtrader in a project:: + + import mql5_zmq_backtrader diff --git a/mql5_zmq_backtrader/__init__.py b/mql5_zmq_backtrader/__init__.py new file mode 100644 index 0000000..0365487 --- /dev/null +++ b/mql5_zmq_backtrader/__init__.py @@ -0,0 +1,11 @@ +""" +Top-level package for mql5_zmq_backtrader. +Project developed to work as a server for Python trading community. It is based on ZeroMQ sockets and uses JSON format to communicate messages. It is a python library for the ZeroMQ API within backtrader framework. It allows rapid trading algo development. For details of API behavior, please see the online API document. +""" + +__author__ = """R. Martin Parrondo""" +__version__ = '0.1.0' + +from .mt5store import * +from .mt5broker import * +from .mt5data import * diff --git a/mql5_zmq_backtrader/adapter.py b/mql5_zmq_backtrader/adapter.py new file mode 100755 index 0000000..109e746 --- /dev/null +++ b/mql5_zmq_backtrader/adapter.py @@ -0,0 +1,34 @@ +import pprint +from datetime import datetime + + +class Adapter(object): + def __init__(self, raw): + self._raw = raw + + def __getattr__(self, key): + if key in self._raw: + val = self._raw[key] + if isinstance(val, (int or float)) and key.endswith('_time'): + return datetime.utcfromtimestamp(val) + else: + return val + return super().__getattribute__(key) + + def __repr__(self): + return '{name}({raw})'.format( + name=self.__class__.__name__, + raw=pprint.pformat(self._raw, indent=4), + ) + + +class BalanceAdapter(Adapter): + pass + + +class OrderAdapter(Adapter): + pass + + +class PositionAdapter(Adapter): + pass diff --git a/mql5_zmq_backtrader/cli.py b/mql5_zmq_backtrader/cli.py new file mode 100644 index 0000000..8e88fd8 --- /dev/null +++ b/mql5_zmq_backtrader/cli.py @@ -0,0 +1,16 @@ +"""Console script for mql5_zmq_backtrader.""" +import sys +import click + + +@click.command() +def main(args=None): + """Console script for mql5_zmq_backtrader.""" + click.echo("Replace this message by putting your code into " + "mql5_zmq_backtrader.cli.main") + click.echo("See click documentation at https://click.palletsprojects.com/") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) # pragma: no cover diff --git a/mql5_zmq_backtrader/mql5_zmq_backtrader.py b/mql5_zmq_backtrader/mql5_zmq_backtrader.py new file mode 100644 index 0000000..dd0b80e --- /dev/null +++ b/mql5_zmq_backtrader/mql5_zmq_backtrader.py @@ -0,0 +1 @@ +"""Main module.""" diff --git a/mql5_zmq_backtrader/mt5broker.py b/mql5_zmq_backtrader/mt5broker.py new file mode 100755 index 0000000..293fcc6 --- /dev/null +++ b/mql5_zmq_backtrader/mt5broker.py @@ -0,0 +1,344 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import collections + +from backtrader import BrokerBase, Order, BuyOrder, SellOrder +from backtrader.utils.py3 import with_metaclass +from backtrader.comminfo import CommInfoBase +from backtrader.position import Position + +from mt5 import mt5store + + +class MTraderCommInfo(CommInfoBase): + def getvaluesize(self, size, price): + # In real life the margin approaches the price + return abs(size) * price + + def getoperationcost(self, size, price): + """Returns the needed amount of cash an operation would cost""" + # Same reasoning as above + return abs(size) * price + + +class MetaMTraderBroker(BrokerBase.__class__): + def __init__(cls, name, bases, dct): + """Class has already been created ... register""" + # Initialize the class + super(MetaMTraderBroker, cls).__init__(name, bases, dct) + mt5store.MTraderStore.BrokerCls = cls + + +class MTraderBroker(with_metaclass(MetaMTraderBroker, BrokerBase)): + """Broker implementation for MetaTrader 5. + + This class maps the orders/positions from MetaTrader to the + internal API of `backtrader`. + + Params: + + - `use_positions` (Ram default:`True`): When connecting to the broker + provider use the existing positions to kickstart the broker. + + Set to `False` during instantiation to disregard any existing + position + """ + # TODO: close positions + + params = ( + ('use_positions', True), + ) + + def __init__(self, **kwargs): + super(MTraderBroker, self).__init__() + + self.o = mt5store.MTraderStore(**kwargs) + + self.orders = collections.OrderedDict() # orders by order id + self.notifs = collections.deque() # holds orders which are notified + + self.opending = collections.defaultdict(list) # pending transmission + self.brackets = dict() # confirmed brackets + + self.startingcash = self.cash = 0.0 + self.startingvalue = self.value = 0.0 + self.positions = collections.defaultdict(Position) + + self.addcommissioninfo(self, MTraderCommInfo(mult=1.0, stocklike=False)) + + def start(self): + super(MTraderBroker, self).start() + self.addcommissioninfo(self, MTraderCommInfo(mult=1.0, stocklike=False)) + self.o.start(broker=self) + # Check MetaTrader account + self.o.check_account() + # Get balance on start + self.o.get_balance() + self.startingcash = self.cash = self.o.get_cash() + self.startingvalue = self.value = self.o.get_value() + + if self.p.use_positions: + for p in self.o.get_positions(): + # print('position for instrument:', p.symbol) + is_sell = p.type.endswith('_SELL') + size = float(p.volume) + if is_sell: + size = -size + price = float(p.open) + self.positions[p.symbol] = Position(size, price) + + def data_started(self, data): + pos = self.getposition(data) + + if pos.size == 0: + return + + if pos.size < 0: + order = SellOrder(data=data, size=pos.size, price=pos.price, + exectype=Order.Market, simulated=True) + elif pos.size > 0: + order = BuyOrder(data=data, size=pos.size, price=pos.price, + exectype=Order.Market, simulated=True) + + order.addcomminfo(self.getcommissioninfo(data)) + order.execute(0, pos.size, pos.price, + 0, 0.0, 0.0, + pos.size, 0.0, 0.0, + 0.0, 0.0, + pos.size, pos.price) + + order.completed() + self.notify(order) + + def stop(self): + super(MTraderBroker, self).stop() + self.o.stop() + + def getcash(self): + # This call cannot block if no answer is available from MTrader + self.cash = cash = self.o.get_cash() + return cash + + def getvalue(self, datas=None): + self.value = self.o.get_value() + return self.value + + def getposition(self, data, clone=True): + # return self.o.getposition(data._dataname, clone=clone) + pos = self.positions[data._dataname] + if clone: + pos = pos.clone() + + return pos + + def orderstatus(self, order): + o = self.orders[order.ref] + return o.status + + def _submit(self, oref): + order = self.orders[oref] + order.submit(self) + self.notify(order) + + def _reject(self, oref): + order = self.orders[oref] + order.reject(self) + self.notify(order) + + def _accept(self, oref): + order = self.orders[oref] + order.accept() + self.notify(order) + + def _cancel(self, oref): + order = self.orders[oref] + order.cancel() + self.notify(order) + self._bracketize(order, cancel=True) + + def _expire(self, oref): + order = self.orders[oref] + order.expire() + self.notify(order) + self._bracketize(order, cancel=True) + + def _bracketize(self, order, cancel=False): + pref = getattr(order.parent, 'ref', order.ref) # parent ref or self + br = self.brackets.pop(pref, None) # to avoid recursion + if br is None: + return + + if not cancel: + if len(br) == 3: # all 3 orders in place, parent was filled + br = br[1:] # discard index 0, parent + for o in br: + o.activate() # simulate activate for children + self.brackets[pref] = br # not done - reinsert children + + elif len(br) == 2: # filling a children + oidx = br.index(order) # find index to filled (0 or 1) + self._cancel(br[1 - oidx].ref) # cancel remaining (1 - 0 -> 1) + else: + # Any cancellation cancel the others + for o in br: + if o.alive(): + self._cancel(o.ref) + + def _fill_external(self, data, size, price): + if size == 0: + return + + pos = self.getposition(data, clone=False) + pos.update(size, price) + + if size < 0: + order = SellOrder(data=data, + size=size, price=price, + exectype=Order.Market, + simulated=True) + else: + order = BuyOrder(data=data, + size=size, price=price, + exectype=Order.Market, + simulated=True) + + order.addcomminfo(self.getcommissioninfo(data)) + order.execute(0, size, price, + 0, 0.0, 0.0, + size, 0.0, 0.0, + 0.0, 0.0, + size, price) + + order.completed() + self.notify(order) + + def _fill(self, oref, size, price, reason, **kwargs): + order = self.orders[oref] + if not order.alive(): # can be a bracket + pref = getattr(order.parent, 'ref', order.ref) + if pref not in self.brackets: + msg = ('Order fill received for {}, with price {} and size {} ' + 'but order is no longer alive and is not a bracket. ' + 'Unknown situation {}') + msg = msg.format(order.ref, price, size, reason) + self.o.put_notification(msg) + return + + # [main, stopside, takeside], neg idx to array are -3, -2, -1 + if reason == 'STOP_LOSS_ORDER': + order = self.brackets[pref][-2] + elif reason == 'TAKE_PROFIT_ORDER': + order = self.brackets[pref][-1] + else: + msg = ('Order fill received for {}, with price {} and size {} ' + 'but order is no longer alive and is a bracket. ' + 'Unknown situation {}') + msg = msg.format(order.ref, price, size, reason) + self.o.put_notification(msg) + return + + data = order.data + pos = self.getposition(data, clone=False) + psize, pprice, opened, closed = pos.update(size, price) + comminfo = self.getcommissioninfo(data) + + closedvalue = closedcomm = 0.0 + openedvalue = openedcomm = 0.0 + margin = pnl = 0.0 + + order.execute(data.datetime[0], size, price, + closed, closedvalue, closedcomm, + opened, openedvalue, openedcomm, + margin, pnl, + psize, pprice) + + if order.executed.remsize: + order.partial() + self.notify(order) + else: + order.completed() + self.notify(order) + self._bracketize(order) + + def _transmit(self, order): + oref = order.ref + pref = getattr(order.parent, 'ref', oref) # parent ref or self + + if order.transmit: + if oref != pref: # children order + # Put parent in orders dict, but add stopside and takeside + # to order creation. Return the takeside order, to have 3s + takeside = order # alias for clarity + parent, stopside = self.opending.pop(pref) + for o in parent, stopside, takeside: + self.orders[o.ref] = o # write them down + + self.brackets[pref] = [parent, stopside, takeside] + self.o.order_create(parent, stopside, takeside) + return takeside # parent was already returned + + else: # Parent order, which is not being transmitted + self.orders[order.ref] = order + return self.o.order_create(order) + + # Not transmitting + self.opending[pref].append(order) + return order + + def buy(self, owner, data, + size, price=None, plimit=None, + exectype=None, valid=None, tradeid=0, oco=None, + trailamount=None, trailpercent=None, + parent=None, transmit=True, + **kwargs): + + #ram + print("mt5broker **kwargs", kwargs) + + order = BuyOrder(owner=owner, data=data, + size=size, price=price, pricelimit=plimit, + exectype=exectype, valid=valid, tradeid=tradeid, + trailamount=trailamount, trailpercent=trailpercent, + parent=parent, transmit=transmit) + + order.addinfo(**kwargs) + order.addcomminfo(self.getcommissioninfo(data)) + return self._transmit(order) + + def sell(self, owner, data, + size, price=None, plimit=None, + exectype=None, valid=None, tradeid=0, oco=None, + trailamount=None, trailpercent=None, + parent=None, transmit=True, + **kwargs): + + order = SellOrder(owner=owner, data=data, + size=size, price=price, pricelimit=plimit, + exectype=exectype, valid=valid, tradeid=tradeid, + trailamount=trailamount, trailpercent=trailpercent, + parent=parent, transmit=transmit) + + order.addinfo(**kwargs) + order.addcomminfo(self.getcommissioninfo(data)) + return self._transmit(order) + + def cancel(self, order): + if not self.orders.get(order.ref, False): + return + if order.status == Order.Cancelled: # already cancelled + return + + return self.o.order_cancel(order) + + def notify(self, order): + self.notifs.append(order.clone()) + + def get_notification(self): + if not self.notifs: + return None + + return self.notifs.popleft() + + def next(self): + self.notifs.append(None) # mark notification boundary \ No newline at end of file diff --git a/mql5_zmq_backtrader/mt5data.py b/mql5_zmq_backtrader/mt5data.py new file mode 100755 index 0000000..89ec9f3 --- /dev/null +++ b/mql5_zmq_backtrader/mt5data.py @@ -0,0 +1,244 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +from datetime import datetime + +from backtrader.feed import DataBase +from backtrader import date2num, num2date +from backtrader.utils.py3 import queue, with_metaclass + +from mt5 import mt5store + + +class MetaMTraderData(DataBase.__class__): + def __init__(cls, name, bases, dct): + """Class has already been created ... register""" + # Initialize the class + super(MetaMTraderData, cls).__init__(name, bases, dct) + + # Register with the store + mt5store.MTraderStore.DataCls = cls + + +class MTraderData(with_metaclass(MetaMTraderData, DataBase)): + """MTrader Data Feed. + + TODO: implement tick data. Main problem is that Backtrader is not tick oriented. + TODO: test backfill_from + + Params: + + - `historical` (default: `False`) + + If set to `True` the data feed will stop after doing the first + download of data. + + The standard data feed parameters `fromdate` and `todate` will be + used as reference. + + - `backfill` (default: `True`) + + Perform backfilling after a disconnection/reconnection cycle. The gap + duration will be used to download the smallest possible amount of data + + - `backfill_from` (default: `None`) + + An additional data source can be passed to do an initial layer of + backfilling. Once the data source is depleted and if requested, + backfilling from IB will take place. This is ideally meant to backfill + from already stored sources like a file on disk, but not limited to. + + - `include_last` (default: `False`) + + Last historical candle is not closed. It will be updated in live stream + + - `reconnect` (default: `True`) + + Reconnect when network connection is down + + """ + params = ( + ('historical', False), # do backfilling at the start + ('backfill', True), # do backfilling when reconnecting + ('backfill_from', None), # additional data source to do backfill from + ('include_last', False), + ('reconnect', True), + ) + + _store = mt5store.MTraderStore + + # States for the Finite State Machine in _load + _ST_FROM, _ST_START, _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(5) + + def islive(self): + """True notifies `Cerebro` that `preloading` and `runonce` + should be deactivated""" + return True + + def __init__(self, **kwargs): + self.o = self._store(**kwargs) + # self._candleFormat = 'bidask' if self.p.bidask else 'midpoint' + + def setenvironment(self, env): + """Receives an environment (cerebro) and passes it over to the store it + belongs to""" + super(MTraderData, self).setenvironment(env) + env.addstore(self.o) + + def start(self): + """Starts the MTrader connection and gets the real contract and + contractdetails if it exists""" + super(MTraderData, self).start() + + # Create attributes as soon as possible + self._statelivereconn = False # if reconnecting in live state + self.qlive = self.o.q_livedata + + #ram + self.contractdetails = None + + self._state = self._ST_OVER + + # Kickstart store and get queue to wait on + self.o.start(data=self) + + # Check if the granularity is supported + data_tf = self.o.get_granularity(self._timeframe, self._compression) + if data_tf is None: + self.put_notification(self.NOTSUPPORTED_TF) + self._state = self._ST_OVER + return + + # Configure server script symbol and time frame + # Error will be raised if params are not supported + #ram self.o.config_server(self.p.dataname, data_tf) + + # Backfill from external data feed + if self.p.backfill_from is not None: + self._state = self._ST_FROM + self.p.backfill_from._start() + else: + self._start_finish() + # initial state for _load + self._state = self._ST_START + self._st_start() + + def _st_start(self): + self.put_notification(self.DELAYED) + + date_begin = num2date( + self.fromdate) if self.fromdate > float('-inf') else None + date_end = num2date( + self.todate) if self.todate < float('inf') else None + + self.qhist = self.o.candles(self.p.dataname, date_begin, date_end, self._timeframe, + self._compression, self.p.include_last) + + self._state = self._ST_HISTORBACK + + return True + + def stop(self): + '''Stops and tells the store to stop''' + super(MTraderData, self).stop() + self.o.stop() + + def haslivedata(self): + return bool(self.qlive) # do not return the obj + + def _load(self): + if self._state == self._ST_OVER: + return False + + while True: + if self._state == self._ST_LIVE: + try: + msg = self.qlive.get() + except queue.Empty: + return None + + if msg: + if msg['status'] == 'DISCONNECTED': + self.put_notification(self.DISCONNECTED) + + if not self.p.backfill: + self._state = self._ST_OVER + + self._statelivereconn = True + continue + + elif msg['status'] == 'CONNECTED' and self._statelivereconn: + self.put_notification(self.CONNECTED) + self._statelivereconn = False + + if len(self) > 1: + self.fromdate = self.lines.datetime[-1] + + self._st_start() + continue + + if self._load_history(msg['data']): + return True # loading worked + + elif self._state == self._ST_HISTORBACK: + msg = self.qhist.get() + if msg is None: + # Situation not managed. Simply bail out + self.put_notification(self.DISCONNECTED) + self._state = self._ST_OVER + return False # error management cancelled the queue + + if msg: + if self._load_history(msg): + return True # loading worked + + continue # not loaded ... date may have been seen + else: + # End of histdata + if self.p.historical: # only historical + self.put_notification(self.DISCONNECTED) + self._state = self._ST_OVER + return False # end of historical + + # Live is also wished - go for it + self._state = self._ST_LIVE + self.put_notification(self.LIVE) + continue + + elif self._state == self._ST_FROM: + if not self.p.backfill_from.next(): + # additional data source is consumed + self._state = self._ST_START + continue + + # copy lines of the same name + for alias in self.lines.getlinealiases(): + lsrc = getattr(self.p.backfill_from.lines, alias) + ldst = getattr(self.lines, alias) + + ldst[0] = lsrc[0] + + return True + + elif self._state == self._ST_START: + if not self._st_start(): + self._state = self._ST_OVER + return False + + def _load_history(self, ohlcv): + time_stamp, _open, _high, _low, _close, _volume = ohlcv + d_time = datetime.utcfromtimestamp(time_stamp) + + dt = date2num(d_time) + # time already seen + if dt <= self.lines.datetime[-1]: + return False + + self.lines.datetime[0] = date2num(d_time) + self.lines.open[0] = _open + self.lines.high[0] = _high + self.lines.low[0] = _low + self.lines.close[0] = _close + self.lines.volume[0] = _volume + self.lines.openinterest[0] = 0.0 + return True diff --git a/mql5_zmq_backtrader/mt5store.py b/mql5_zmq_backtrader/mt5store.py new file mode 100755 index 0000000..500f39d --- /dev/null +++ b/mql5_zmq_backtrader/mt5store.py @@ -0,0 +1,715 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import zmq +import collections +from datetime import datetime +import threading + +from mt5.adapter import PositionAdapter, OrderAdapter, BalanceAdapter + +import backtrader as bt +from backtrader.metabase import MetaParams +from backtrader.utils.py3 import queue, with_metaclass +import sys + + +class MTraderError(Exception): + def __init__(self, *args, **kwargs): + default = 'Meta Trader 5 ERROR' + if not (args or kwargs): + args = (default) + super(MTraderError, self).__init__(*args, **kwargs) + + +class ServerConfigError(MTraderError): + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + + +class ServerDataError(MTraderError): + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + + +class TimeFrameError(MTraderError): + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + + +class StreamError(MTraderError): + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + + +class MTraderAPI: + """ + This class implements Python side for MQL5 JSON API + See https://github.com/khramkov/MQL5-JSON-API for docs + """ + # TODO: unify error handling + + def __init__(self, host=None): + self.HOST = host or 'localhost' + self.SYS_PORT = 15555 # REP/REQ port + self.DATA_PORT = 15556 # PUSH/PULL port + self.LIVE_PORT = 15557 # PUSH/PULL port + self.EVENTS_PORT = 15558 # PUSH/PULL port + + # ZeroMQ timeout in miliseconds + self.SYS_TIMEOUT = 1000 + self.DATA_TIMEOUT = 10000 + self.REQUEST_RETRIES = 3 # Lazy Pirate implementation + self.sequence = 0 # Lazy Pirate request sequence + # initialise ZMQ context + self.context = zmq.Context() + + # connect to server sockets + try: + self.sys_socket = self.context.socket(zmq.REQ) + # set port timeout + self.sys_socket.RCVTIMEO = self.SYS_TIMEOUT + self.sys_socket.connect( + 'tcp://{}:{}'.format(self.HOST, self.SYS_PORT)) + # Lazy Pirate implementation + self.poll = zmq.Poller() + self.poll.register(self.sys_socket, zmq.POLLIN) + + self.data_socket = self.context.socket(zmq.PULL) + # set port timeout + self.data_socket.RCVTIMEO = self.DATA_TIMEOUT + self.data_socket.connect( + 'tcp://{}:{}'.format(self.HOST, self.DATA_PORT)) + except zmq.ZMQError: + raise zmq.ZMQBindError("Binding ports ERROR") + + def _send_request(self, data: dict) -> None: + """Send request to server via ZeroMQ System socket + Lazy Pirate implementation. + """ + # ram Caller's name + print("Caller 2 ", sys._getframe(2).f_code.co_name) + + try: + # ram sequence = 0 + retries_left = self.REQUEST_RETRIES + while retries_left: + self.sequence += 1 + request = str(self.sequence).encode() + print("I: Sending (%s)" % self.sequence) + print("data ", data) + self.sys_socket.send_json(data) + + expect_reply = True + while expect_reply: + socks = dict(self.poll.poll(self.SYS_TIMEOUT)) + if socks.get(self.sys_socket) == zmq.POLLIN: + msg = self.sys_socket.recv_string() + if not msg: + break + # terminal received the request + if str(msg) == 'OK': + print("I: Server replied %s" % msg) + retries_left = 0 + expect_reply = False + else: + print("E: Malformed reply from server: %s" % msg) + + else: + print("W: No response from server, retrying…") + # Socket is confused. Close and remove it. + self.sys_socket.setsockopt(zmq.LINGER, 0) + self.sys_socket.close() + self.poll.unregister(self.sys_socket) + retries_left -= 1 + if retries_left == 0: + print("E: Server seems to be offline, abandoning") + break + print("I: Reconnecting and resending (%s)" % + self.sequence) + # Create new connection + self.sys_socket = self.context.socket(zmq.REQ) + self.sys_socket.RCVTIMEO = self.SYS_TIMEOUT + self.sys_socket.connect( + 'tcp://{}:{}'.format(self.HOST, self.SYS_PORT)) + self.poll.register(self.sys_socket, zmq.POLLIN) + self.sys_socket.send_json(data) + + # ram self.context.term() + except zmq.ZMQError: + raise zmq.NotDone("Sending request ERROR") + + def _pull_reply(self): + # Get reply from server via Data socket with timeout + try: + msg = self.data_socket.recv_json() + #ram except zmq.ZMQError: + #ram raise zmq.NotDone('Data socket timeout ERROR') + except zmq.Again as e: + return None + except zmq.ZMQError as e: + logger.debug("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e)) + return msg + + def live_socket(self, context=None): + """Connect to socket in a ZMQ context""" + try: + context = context or zmq.Context.instance() + socket = context.socket(zmq.PULL) + socket.connect('tcp://{}:{}'.format(self.HOST, self.LIVE_PORT)) + except zmq.ZMQError: + raise zmq.ZMQBindError("Live port connection ERROR") + return socket + + def streaming_socket(self, context=None): + """Connect to socket in a ZMQ context""" + try: + context = context or zmq.Context.instance() + socket = context.socket(zmq.PULL) + socket.connect('tcp://{}:{}'.format(self.HOST, self.EVENTS_PORT)) + except zmq.ZMQError: + raise zmq.ZMQBindError("Data port connection ERROR") + return socket + + def construct_and_send(self, **kwargs) -> dict: + """Construct a request dictionary from default and send it to server""" + + # default dictionary + request = { + "action": None, + "actionType": None, + "symbol": None, + "chartTF": None, + "fromDate": None, + "toDate": None, + "id": None, + "magic": 1234, + "volume": None, + "price": None, + "stoploss": None, + "takeprofit": None, + "expiration": None, + "deviation": None, + "comment": None + } + + # update dict values if exist + for key, value in kwargs.items(): + if key in request: + request[key] = value + else: + raise KeyError('Unknown key in **kwargs ERROR') + + # send dict to server + self._send_request(request) + + # return server reply + return self._pull_reply() + + +class MetaSingleton(MetaParams): + """Metaclass to make a metaclassed class a singleton""" + def __init__(cls, name, bases, dct): + super(MetaSingleton, cls).__init__(name, bases, dct) + cls._singleton = None + + def __call__(cls, *args, **kwargs): + if cls._singleton is None: + cls._singleton = ( + super(MetaSingleton, cls).__call__(*args, **kwargs)) + + return cls._singleton + + +class MTraderStore(with_metaclass(MetaSingleton, object)): + """ + Singleton class wrapping to control the connections to MetaTrader. + + Balance update occurs at the beginning and after each + transaction registered by '_t_streaming_events'. + """ + + # TODO: implement stop_limit + # TODO: Check position ticket + + BrokerCls = None # broker class will autoregister + DataCls = None # data class will auto register + + params = () + + # The Unix epoch (or Unix time or POSIX time or Unix timestamp) + _DTEPOCH = datetime(1970, 1, 1) + + # MTrader supported granularities + _GRANULARITIES = { + # (bt.TimeFrame.Ticks, 1): 'Ticks', + (bt.TimeFrame.Minutes, 1): 'M1', + (bt.TimeFrame.Minutes, 2): 'M2', + (bt.TimeFrame.Minutes, 3): 'M3', + (bt.TimeFrame.Minutes, 4): 'M4', + (bt.TimeFrame.Minutes, 5): 'M5', + (bt.TimeFrame.Minutes, 6): 'M6', + (bt.TimeFrame.Minutes, 10): 'M10', + (bt.TimeFrame.Minutes, 12): 'M12', + (bt.TimeFrame.Minutes, 15): 'M15', + (bt.TimeFrame.Minutes, 20): 'M20', + (bt.TimeFrame.Minutes, 30): 'M30', + (bt.TimeFrame.Minutes, 60): 'H1', + (bt.TimeFrame.Minutes, 120): 'H2', + (bt.TimeFrame.Minutes, 180): 'H3', + (bt.TimeFrame.Minutes, 240): 'H4', + (bt.TimeFrame.Minutes, 360): 'H6', + (bt.TimeFrame.Minutes, 480): 'H8', + (bt.TimeFrame.Minutes, 720): 'H12', + (bt.TimeFrame.Days, 1): 'D1', + (bt.TimeFrame.Weeks, 1): 'W1', + (bt.TimeFrame.Months, 1): 'MN1', + } + + # Order type matching with MetaTrader 5 + _ORDEREXECS = { + # Market Buy order + (bt.Order.Market, 'buy'): 'ORDER_TYPE_BUY', + # Market Sell order + (bt.Order.Market, 'sell'): 'ORDER_TYPE_SELL', + # Buy Limit pending order + (bt.Order.Limit, 'buy'): 'ORDER_TYPE_BUY_LIMIT', + # Sell Limit pending order + (bt.Order.Limit, 'sell'): 'ORDER_TYPE_SELL_LIMIT', + # Buy Stop pending order + (bt.Order.Stop, 'buy'): 'ORDER_TYPE_BUY_STOP', + # Sell Stop pending order + (bt.Order.Stop, 'sell'): 'ORDER_TYPE_SELL_STOP', + # Upon reaching the order price, a pending Buy Limit + (bt.Order.StopLimit, 'buy'): 'ORDER_TYPE_BUY_STOP_LIMIT', + # order is placed at the StopLimit price + # Upon reaching the order price, a pending Sell Limit + (bt.Order.StopLimit, 'sell'): 'ORDER_TYPE_SELL_STOP_LIMIT', + # order is placed at the StopLimit price + } + + @classmethod + def getdata(cls, *args, **kwargs): + """Returns `DataCls` with args, kwargs""" + return cls.DataCls(*args, **kwargs) + + @classmethod + def getbroker(cls, *args, **kwargs): + """Returns broker with *args, **kwargs from registered `BrokerCls`""" + return cls.BrokerCls(*args, **kwargs) + + def __init__(self, host='localhost'): + super(MTraderStore, self).__init__() + + self.notifs = collections.deque() # store notifications for cerebro + + self._env = None # reference to cerebro for general notifications + self.broker = None # broker instance + self.datas = list() # datas that have registered over start + + self._orders = collections.OrderedDict() # map order.ref to oid + self._ordersrev = collections.OrderedDict() # map oid to order.ref + self._orders_type = dict() # keeps order types + + self.oapi = MTraderAPI(host) + + self._cash = 0.0 + self._value = 0.0 + + self.q_livedata = queue.Queue() + + self._cancel_flag = False + + self.debug = True + + def start(self, data=None, broker=None): + # Datas require some processing to kickstart data reception + if data is None and broker is None: + self.cash = None + return + + if data is not None: + self._env = data._env + # For datas simulate a queue with None to kickstart co + self.datas.append(data) + + if self.broker is not None: + self.broker.data_started(data) + + elif broker is not None: + self.broker = broker + self.broker_threads() + self.streaming_events() + + def stop(self): + # signal end of thread + if self.broker is not None: + self.q_ordercreate.put(None) + self.q_orderclose.put(None) + + def put_notification(self, msg, *args, **kwargs): + self.notifs.append((msg, args, kwargs)) + + def get_notifications(self): + """Return the pending "store" notifications""" + self.notifs.append(None) # put a mark / threads could still append + return [x for x in iter(self.notifs.popleft, None)] + + def get_positions(self): + positions = self.oapi.construct_and_send(action="POSITIONS") + # Error handling + # if positions["error"]: + # raise ServerDataError(positions) + pos_list = positions.get('positions', []) + if self.debug: + print('Open positions: {}.'.format(pos_list)) + return [PositionAdapter(o) for o in pos_list] + + def get_granularity(self, timeframe, compression): + granularity = self._GRANULARITIES.get((timeframe, compression), None) + if granularity is None: + raise ValueError("Metatrader 5 doesn't support frame %s with compression %s" % + (bt.TimeFrame.getname(timeframe), compression)) + return granularity + + def get_cash(self): + return self._cash + + def get_value(self): + return self._value + + def get_balance(self): + try: + bal = self.oapi.construct_and_send(action="BALANCE") + except Exception as e: + self.put_notification(e) + # TODO: error handling + # if bal['error']: + # self.put_notification(bal) + # continue + try: + self._cash = float(bal["balance"]) + self._value = float(bal["equity"]) + except KeyError as e: + #ram + self.put_notification(e) + pass + + def streaming_events(self): + t = threading.Thread(target=self._t_livedata, daemon=True) + t.start() + + t = threading.Thread(target=self._t_streaming_events, daemon=True) + t.start() + + def _t_livedata(self): + # create socket connection for the Thread + socket = self.oapi.live_socket() + while True: + try: + last_candle = socket.recv_json() + except zmq.ZMQError: + raise zmq.NotDone("Live data ERROR") + + self.q_livedata.put(last_candle) + + def _t_streaming_events(self): + # create socket connection for the Thread + socket = self.oapi.streaming_socket() + while True: + try: + transaction = socket.recv_json() + except zmq.ZMQError: + raise zmq.NotDone("Streaming data ERROR") + + self._transaction(transaction) + + def broker_threads(self): + self.q_ordercreate = queue.Queue() + t = threading.Thread(target=self._t_order_create, daemon=True) + t.start() + + self.q_orderclose = queue.Queue() + t = threading.Thread(target=self._t_order_cancel, daemon=True) + t.start() + + def order_create(self, order, stopside=None, takeside=None, **kwargs): + """Creates an order""" + okwargs = dict() + okwargs['action'] = 'TRADE' + + side = 'buy' if order.isbuy() else 'sell' + order_type = self._ORDEREXECS.get((order.exectype, side), None) + if order_type is None: + raise ValueError("Wrong order type: %s or side: %s" % + (order.exectype, side)) + + okwargs['actionType'] = order_type + okwargs['symbol'] = order.data._dataname + okwargs['volume'] = abs(order.created.size) + + if order.exectype != bt.Order.Market: + okwargs['price'] = format(order.created.price) + + if order.valid is None: + okwargs['expiration'] = 0 # good to cancel + else: + okwargs['expiration'] = order.valid # good to date + + if order.exectype == bt.Order.StopLimit: + okwargs['price'] = order.created.pricelimit + + # TODO: implement StopTrail + # if order.exectype == bt.Order.StopTrail: + # okwargs['distance'] = order.trailamount + + okwargs['comment'] = dict() + + if stopside is not None and stopside.price is not None: + okwargs['stoploss'] = stopside.price + okwargs['comment']['stopside'] = stopside.ref + + if takeside is not None and takeside.price is not None: + okwargs['takeprofit'] = takeside.price + okwargs['comment']['takeside'] = takeside.ref + + # set store backtrader order ref as MT5 order magic number + try: + okwargs['magic'] = order.info["magic"] #Ram Magic number must be inmutable + except KeyError: + print(KeyError) + + okwargs.update(**kwargs) # anything from the user + self.q_ordercreate.put((order.ref, okwargs,)) + + # notify orders of being submitted + self.broker._submit(order.ref) + if stopside is not None and stopside.price is not None: + self.broker._submit(stopside.ref) + if takeside is not None and takeside.price is not None: + self.broker._submit(takeside.ref) + + return order + + def _t_order_create(self): + while True: + msg = self.q_ordercreate.get() + if msg is None: + break + + oref, okwargs = msg + + try: + o = self.oapi.construct_and_send(**okwargs) + except Exception as e: + self.put_notification(e) + self.broker._reject(oref) + return + + if self.debug: + print(o) + + if o['error']: + self.put_notification(o['description']) + self.broker._reject(oref) + return + else: + oid = o['order'] + + self._orders[oref] = oid + self.broker._submit(oref) + + # keeps orders types + self._orders_type[oref] = okwargs['actionType'] + # maps ids to backtrader order + self._ordersrev[oid] = oref + + def order_cancel(self, order): + self.q_orderclose.put(order.ref) + return order + + def _t_order_cancel(self): + while True: + oref = self.q_orderclose.get() + if oref is None: + break + + oid = self._orders.get(oref, None) + if oid is None: + continue # the order is no longer there + + # get symbol name + order = self.broker.orders[oref] + symbol = order.data._dataname + # get order type + order_type = self._orders_type.get(oref, None) + + try: + if order_type in ['ORDER_TYPE_BUY', 'ORDER_TYPE_SELL']: + self.close_position(oid, symbol) + else: + self.cancel_order(oid, symbol) + except Exception as e: + self.put_notification( + "Order not cancelled: {}, {}".format(oid, e)) + continue + + self._cancel_flag = True + self.broker._cancel(oref) + + def candles(self, dataname, dtbegin, dtend, timeframe, compression, include_first=False): + tf = self.get_granularity(timeframe, compression) + + begin = end = None + if dtbegin: + begin = int((dtbegin - self._DTEPOCH).total_seconds()) + if dtend: + end = int((dtbegin - self._DTEPOCH).total_seconds()) + + if self.debug: + print('Fetching: {}, Timeframe: {}, Fromdate: {}'.format( + dataname, tf, dtbegin)) + + data = self.oapi.construct_and_send(action="HISTORY", actionType="DATA", symbol=dataname, + chartTF=tf, fromDate=begin, toDate=end) + candles = data['data'] + # Remove last unclosed candle + if not include_first: + try: + del candles[-1] + except: + pass + + q = queue.Queue() + for c in candles: + q.put(c) + + q.put({}) + return q + + '''ram + def config_server(self, symbol: str, timeframe: str) -> None: + """Set server terminal symbol and time frame""" + conf = self.oapi.construct_and_send(action="CONFIG", symbol=symbol, chartTF=timeframe) + + # TODO Error + # Error handling + if conf["error"]: + print(conf) + if conf["description"] == "Wrong symbol dosn't exist": + raise ServerConfigError("Symbol dosn't exist") + self.put_notification(conf["description"]) + ''' + + def check_account(self) -> None: + """Get MetaTrader 5 account settings""" + # ram Caller's name + print("Caller 3 ", sys._getframe(2).f_code.co_name) + + conf = self.oapi.construct_and_send(action="ACCOUNT") + + # Error handling + if conf["error"]: + raise ServerDataError(conf) + + for key, value in conf.items(): + print(key, value, sep=' - ') + + def close_position(self, oid, symbol): + if self.debug: + print('Closing position: {}, on symbol: {}'.format(oid, symbol)) + + conf = self.oapi.construct_and_send( + action="TRADE", actionType='POSITION_CLOSE_ID', symbol=symbol, id=oid) + print(conf) + # Error handling + if conf["error"]: + raise ServerDataError(conf) + + def cancel_order(self, oid, symbol): + if self.debug: + print('Cancelling order: {}, on symbol: {}'.format(oid, symbol)) + + conf = self.oapi.construct_and_send( + action="TRADE", actionType='ORDER_CANCEL', symbol=symbol, id=oid) + print(conf) + # Error handling + if conf["error"]: + raise ServerDataError(conf) + + def _transaction(self, trans): + # Invoked from Streaming Events. May actually receive an event for an + # oid which has not yet been returned after creating an order. Hence + # store if not yet seen, else forward to processer + + oid = oref = None + + try: + request, reply = trans.values() + except KeyError: + raise KeyError(trans) + + # Update balance after transaction + # self.get_balance() + + if self.debug: + print(request, reply, sep='\n') + + if request['action'] == 'TRADE_ACTION_DEAL': + # get order id (matches transaction id) + oid = request['order'] + elif request['action'] == 'TRADE_ACTION_PENDING': + oid = request['order'] + + elif request['action'] == 'TRADE_ACTION_SLTP': + pass + + elif request['action'] == 'TRADE_ACTION_MODIFY': + pass + + elif request['action'] == 'TRADE_ACTION_REMOVE': + pass + + elif request['action'] == 'TRADE_ACTION_CLOSE_BY': + pass + else: + return + + # try: + # oref = self._ordersrev.pop(oid) + # except KeyError: + # raise KeyError(oid) + + if oid in self._orders.values(): + # when an order id exists process transaction + self._process_transaction(oid, request, reply) + else: + # external order created this transaction + if self._cancel_flag and reply['result'] == 'TRADE_RETCODE_DONE': + self._cancel_flag = False + + size = float(reply['volume']) + price = float(reply['price']) + if request['type'].endswith('_SELL'): + size = -size + for data in self.datas: + if data._name == request['symbol']: + self.broker._fill_external(data, size, price) + break + + def _process_transaction(self, oid, request, reply): + try: + # get a reference to a backtrader order based on the order id / trade id + oref = self._ordersrev[oid] + except KeyError: + return + + if request['action'] == 'TRADE_ACTION_PENDING': + pass + + if reply['result'] == 'TRADE_RETCODE_DONE': + size = float(reply['volume']) + price = float(reply['price']) + if request['type'].endswith('_SELL'): + size = -size + self.broker._fill(oref, size, price, reason=request['type']) diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..3439c45 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,10 @@ +pip==19.2.3 +bump2version==0.5.11 +wheel==0.33.6 +watchdog==0.9.0 +flake8==3.7.8 +tox==3.14.0 +coverage==4.5.4 +Sphinx==1.8.5 +twine==1.14.0 +Click==7.0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..e996d7a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,22 @@ +[bumpversion] +current_version = 0.1.0 +commit = True +tag = True + +[bumpversion:file:setup.py] +search = version='{current_version}' +replace = version='{new_version}' + +[bumpversion:file:mql5_zmq_backtrader/__init__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bdist_wheel] +universal = 1 + +[flake8] +exclude = docs + +[aliases] +# Define setup.py command aliases here + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..74fc99a --- /dev/null +++ b/setup.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +"""The setup script.""" + +from setuptools import setup, find_packages + +with open('README.rst') as readme_file: + readme = readme_file.read() + +with open('HISTORY.rst') as history_file: + history = history_file.read() + +requirements = ['Click>=7.0', ] + +setup_requirements = [ ] + +test_requirements = [ ] + +setup( + author="R. Martin Parrondo", + author_email='audreyr@example.com', + python_requires='>=3.5', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + ], + description="Project developed to work as a server for Python trading community. It is based on ZeroMQ sockets and uses JSON format to communicate messages. It is a python library for the ZeroMQ API within backtrader framework. It allows rapid trading algo development. For details of API behavior, please see the online API document.", + entry_points={ + 'console_scripts': [ + 'mql5_zmq_backtrader=mql5_zmq_backtrader.cli:main', + ], + }, + install_requires=requirements, + license="MIT license", + long_description=readme + '\n\n' + history, + include_package_data=True, + keywords='mql5_zmq_backtrader', + name='mql5_zmq_backtrader', + packages=find_packages(include=['mql5_zmq_backtrader', 'mql5_zmq_backtrader.*']), + setup_requires=setup_requirements, + test_suite='tests', + tests_require=test_requirements, + url='https://github.com/parrondo/mql5_zmq_backtrader', + version='0.1.0', + zip_safe=False, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..3c78f4a --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Unit test package for mql5_zmq_backtrader.""" diff --git a/tests/test_mql5_zmq_backtrader.py b/tests/test_mql5_zmq_backtrader.py new file mode 100644 index 0000000..e6b5045 --- /dev/null +++ b/tests/test_mql5_zmq_backtrader.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +"""Tests for `mql5_zmq_backtrader` package.""" + + +import unittest +from click.testing import CliRunner + +from mql5_zmq_backtrader import mql5_zmq_backtrader +from mql5_zmq_backtrader import cli + + +class TestMql5_zmq_backtrader(unittest.TestCase): + """Tests for `mql5_zmq_backtrader` package.""" + + def setUp(self): + """Set up test fixtures, if any.""" + + def tearDown(self): + """Tear down test fixtures, if any.""" + + def test_000_something(self): + """Test something.""" + + def test_command_line_interface(self): + """Test the CLI.""" + runner = CliRunner() + result = runner.invoke(cli.main) + assert result.exit_code == 0 + assert 'mql5_zmq_backtrader.cli.main' in result.output + help_result = runner.invoke(cli.main, ['--help']) + assert help_result.exit_code == 0 + assert '--help Show this message and exit.' in help_result.output diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f7e94cb --- /dev/null +++ b/tox.ini @@ -0,0 +1,20 @@ +[tox] +envlist = py35, py36, py37, py38, flake8 + +[travis] +python = + 3.8: py38 + 3.7: py37 + 3.6: py36 + 3.5: py35 + +[testenv:flake8] +basepython = python +deps = flake8 +commands = flake8 mql5_zmq_backtrader tests + +[testenv] +setenv = + PYTHONPATH = {toxinidir} + +commands = python setup.py test