diff --git a/.env.example b/.env.example index 7c071c0..6968c4f 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,65 @@ PROJECT_NAME=FonXLog +#Monolog\Logger::DEBUG = 100 +FINXLOG_DEBUG=100 +FINXLOG_DOMAIN=localhost -FINXLOG_QUOTATION_SERVER_ADDRESS=198.211.118.180 +FINXLOG_QUOTATION_SERVER_ADDRESS=178.62.145.164 +#FINXLOG_QUOTATION_SERVER_ADDRESS=198.211.118.180 FINXLOG_QUOTATION_SERVER_PORT=10000 FINXLOG_AMQP_SERVER_ADDRESS=127.0.0.1 #FINXLOG_AMQP_SERVER_PORT= -FINXLOG_AMQP_TUBE_QUOTATION=finxlog_quotation -FINXLOG_AMQP_TUBE_QUOTATION_FAIL=finxlog_quotation_fail +FINXLOG_AMQP_TUBE_IMPORT=finxlog_quotation +FINXLOG_AMQP_TUBE_IMPORT_FAIL=finxlog_quotation_fail +FINXLOG_AMQP_TUBE_WS=finxlog_ws -FINXLOG_ELASTICO_PARAM={"host":"localhost"} +FINXLOG_ELASTICA_PARAM={"host":"localhost"} +#FINXLOG_ELASTICA_LOG_PARAM={"host":"localhost"} + +FINXLOG_IMPORT_FILTER_OTHER=0 + +FINXLOG_ENV_JS="PROJECT_NAME FINXLOG_DOMAIN FINXLOG_WEBSOCKET_EXTARNAL_HOST FINXLOG_WEBSOCKET_EXTARNAL_PORT FINXLOG_WEBSOCKET_EXTARNAL_PATH FINXLOG_DEBUG" + + + +##### WEBSOCKET BLOCK +#please, always use +# 1. not LISTEN public interface (use 127.0.0.1 or internal) +# 2. setup http proxy on vhost to websocket. nginx: http://nginx.org/ru/docs/http/websocket.html +# //DOMAIN/ws => ws://LISTEN_INTERFACE:LISTEN_PORT/____NOT____SERVICE_PATH +# //localhost/ws => ws://127.0.0.1:8080/ws + +##### -------------------------------- +###DEVELOPER MODE. is just for quick start. do not use +#is direct access for send any data to any users +FINXLOG_WEBSOCKET_LISTEN_INTERFACE=0.0.0.0 +FINXLOG_WEBSOCKET_EXTARNAL_PORT=8080 +FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP=^127\. + +###PRODUCTION MODE: +#FINXLOG_WEBSOCKET_LISTEN_INTERFACE=127.0.0.1 +#default EXTARNAL_PORT=80 + +###PRODUCTION CLOUD MODE with 10.0.3.0 intranet +#FINXLOG_WEBSOCKET_LISTEN_INTERFACE=10.0.0.0 +#FINXLOG_WEBSOCKET_INTERNAL_HOST=10.0.3.x +#default EXTARNAL_PORT=80 +##### -------------------------------- + + + + +##### WEBSOCKET TYPICAL BLOCK +#required for internal service +FINXLOG_WEBSOCKET_LISTEN_PORT=8080 + +#markers for proxy: path or domain +FINXLOG_WEBSOCKET_EXTARNAL_PATH=/ws +#default: FINXLOG_DOMAIN +#FINXLOG_WEBSOCKET_EXTARNAL_HOST=localhost + +#if empty(FILTER_ADDR_REGEXP) and proxy not acceptable. randomize it: for same secure +FINXLOG_WEBSOCKET_SERVICE_PATH=/service + +#one of: Ratchet or ElephantIO. just for performance test +#FINXLOG_WEBSOCKET_SERVICE_LIB=Ratchet diff --git a/.gitignore b/.gitignore index 2206342..ef4215d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ vendor/ composer.lock .env +www/js/config_autobuild.js diff --git a/command/daemon/import/quotation_amqp2db.php b/command/daemon/import/quotation_amqp2db.php new file mode 100755 index 0000000..0c31915 --- /dev/null +++ b/command/daemon/import/quotation_amqp2db.php @@ -0,0 +1,21 @@ +#!/usr/bin/php +setQueueConnector($import->getFailQueueConnector()); +} +$count = null; +foreach ($argv as $a) { + if (is_numeric($a) && $a > 0) { + $count = $a; + } +} + +$import->run($count); diff --git a/command/daemon/quotation_exchange2amqp.php b/command/daemon/import/quotation_exchange2amqp.php similarity index 56% rename from command/daemon/quotation_exchange2amqp.php rename to command/daemon/import/quotation_exchange2amqp.php index bec1d34..c75cef5 100755 --- a/command/daemon/quotation_exchange2amqp.php +++ b/command/daemon/import/quotation_exchange2amqp.php @@ -4,9 +4,9 @@ /** * Always on service: load quotation from source */ -require_once __DIR__ . '/../../vendor/autoload.php'; +require_once __DIR__ . '/../../../vendor/autoload.php'; -$import = new FinXLog\Module\Import\LoadQuotation(); +$import = new FinXLog\Module\ImportQuotation\LoadQuotation(); $import->run( !empty($argv[1]) diff --git a/command/daemon/import/quotation_exchange2amqp.sh b/command/daemon/import/quotation_exchange2amqp.sh new file mode 100755 index 0000000..f639348 --- /dev/null +++ b/command/daemon/import/quotation_exchange2amqp.sh @@ -0,0 +1,14 @@ +#!/bin/bash +FINXLOG_ROOT_DIR=`dirname $0`/../../../; +#import .env +export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') + +# check: +# export | grep FIN + +# netcat > amqp +while : +do + netcat $FINXLOG_QUOTATION_SERVER_ADDRESS $FINXLOG_QUOTATION_SERVER_PORT \ + | xargs -I {} beanstool put -t=$FINXLOG_AMQP_TUBE_IMPORT -b {} +done diff --git a/command/daemon/quotation_exchange2db.php b/command/daemon/import/quotation_exchange2db.php similarity index 63% rename from command/daemon/quotation_exchange2db.php rename to command/daemon/import/quotation_exchange2db.php index 90c9d96..5e3f32c 100755 --- a/command/daemon/quotation_exchange2db.php +++ b/command/daemon/import/quotation_exchange2db.php @@ -4,9 +4,9 @@ /** * Always on service: load quotation from source */ -require_once __DIR__ . '/../../vendor/autoload.php'; +require_once __DIR__ . '/../../../vendor/autoload.php'; -$import = new FinXLog\Module\Import\LoadQuotation(); +$import = new FinXLog\Module\ImportQuotation\LoadQuotation(); $import ->setWorkWithSwitch(false) diff --git a/command/daemon/import/readme.md b/command/daemon/import/readme.md new file mode 100644 index 0000000..e94c7cc --- /dev/null +++ b/command/daemon/import/readme.md @@ -0,0 +1,6 @@ +# Description +this daemons for import exchange quotations + +daemon: ex2db +or +daemon: ex2amqp->AMQP->amqp2db \ No newline at end of file diff --git a/command/daemon/quotation_amqp2db.php b/command/daemon/quotation_amqp2db.php deleted file mode 100755 index 6cf0165..0000000 --- a/command/daemon/quotation_amqp2db.php +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/php -run( - !empty($argv[1]) - ? $argv[1] - : null -); diff --git a/command/daemon/quotation_exchange2amqp.sh b/command/daemon/quotation_exchange2amqp.sh deleted file mode 100755 index c65b236..0000000 --- a/command/daemon/quotation_exchange2amqp.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -FINXLOG_ROOT_DIR=`dirname $0`/../../; -#import .env -export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') - -#export | grep FIN - -# telnet > amqp -telnet $FINXLOG_QUOTATION_SERVER_ADDRESS $FINXLOG_QUOTATION_SERVER_PORT | xargs -I {} echo beanstool put -t=$FINXLOG_AMQP_TUBE -b {} diff --git a/command/daemon/ws/quotation_amqp2ws.php b/command/daemon/ws/quotation_amqp2ws.php new file mode 100755 index 0000000..a6da486 --- /dev/null +++ b/command/daemon/ws/quotation_amqp2ws.php @@ -0,0 +1,6 @@ +#!/usr/bin/php +run(); diff --git a/command/daemon/ws/quotation_ws2client.php b/command/daemon/ws/quotation_ws2client.php new file mode 100755 index 0000000..551048a --- /dev/null +++ b/command/daemon/ws/quotation_ws2client.php @@ -0,0 +1,6 @@ +#!/usr/bin/php +run(); \ No newline at end of file diff --git a/command/daemon/ws/readme.md b/command/daemon/ws/readme.md new file mode 100644 index 0000000..0c23e36 --- /dev/null +++ b/command/daemon/ws/readme.md @@ -0,0 +1,37 @@ +# Description +this daemons to work with Browser WebSocket request from JS_APP + +## 1st part + +get quotations list(or agg like doji) + +JS_App +->WebSocket (client) +->daemon: ws2client (listen) +->AMQP +->daemon: amqp2ws (listen) +->Elasticsearch +//back + ->WebSocket (service) + ->daemon: ws2client (listen) + ->WebSocket (all) + ->JS_App + ->table + ->JS_HighCharts + + + +@todo: +2nd part +daemon: +import/ex2amqp (listen) +->AMQP +->import/amqp2db + ->2db + ->WebSocket (service) + ->daemon: ws2client (listen) + ->WebSocket (update) + ->JS_App + ->table + ->JS_HighCharts + \ No newline at end of file diff --git a/command/setup/web_config_build.php b/command/setup/web_config_build.php new file mode 100755 index 0000000..3c2f734 --- /dev/null +++ b/command/setup/web_config_build.php @@ -0,0 +1,27 @@ +#!/usr/bin/php +getAAPL('BTCUSD', 'day') +); \ No newline at end of file diff --git a/command/test/ws/add_amqp_ws_subscribe.php b/command/test/ws/add_amqp_ws_subscribe.php new file mode 100755 index 0000000..0a91e9d --- /dev/null +++ b/command/test/ws/add_amqp_ws_subscribe.php @@ -0,0 +1,15 @@ +#!/usr/bin/php +put([ + 'quotation' => 'BTCUSD' + ]) + ->put([ + 'quotation' => 'BTCUSD', + 'agg_type' => 'AAPL', + 'agg_period' => 'M1', + ]); \ No newline at end of file diff --git a/command/test/ws/add_amqp_ws_subscribe.sh b/command/test/ws/add_amqp_ws_subscribe.sh new file mode 100755 index 0000000..37872fa --- /dev/null +++ b/command/test/ws/add_amqp_ws_subscribe.sh @@ -0,0 +1,7 @@ +#!/bin/bash +FINXLOG_ROOT_DIR=`dirname $0`/../../../; +#import .env +export $(cat $FINXLOG_ROOT_DIR.env | grep -P '^FINXLOG_(QUOTATION|AMQP)') + +beanstool put -t=$FINXLOG_AMQP_TUBE_WS -b '{"quotation":"BTCUSD"}'; +beanstool put -t=$FINXLOG_AMQP_TUBE_WS -b '{"quotation":"BTCUSD","agg_type":"AAPL","agg_period":"M1"}'; diff --git a/command/test/ws/add_ws_msg_client.php b/command/test/ws/add_ws_msg_client.php new file mode 100755 index 0000000..8c0cc66 --- /dev/null +++ b/command/test/ws/add_ws_msg_client.php @@ -0,0 +1,38 @@ +#!/usr/bin/php +then( + function(\Ratchet\Client\WebSocket $conn) + use ($listen_count) + { + $conn->on( + 'message', + function($msg) use ($conn, $listen_count) + { + echo "\nReceived: " . substr($msg, 0, 100). "...\n\n"; + if ($listen_count) { + static $count = 0; + if (++$count >= $listen_count) { + $conn->close(); + } + } + } + ); + + $conn->send('{"type": "quotations" }'); + $conn->send('{"type":"subscribe","quotation":"_ALL"}'); + $conn->send('{"type":"subscribe","quotation":"BTCUSD"}'); + $conn->send('{"type":"subscribe","quotation":"BTCUSD","agg_period": "M1","agg_type":"AAPL"}'); + } + ); \ No newline at end of file diff --git a/command/test/ws/add_ws_msg_service.php b/command/test/ws/add_ws_msg_service.php new file mode 100755 index 0000000..3e99d3b --- /dev/null +++ b/command/test/ws/add_ws_msg_service.php @@ -0,0 +1,43 @@ +#!/usr/bin/php +addWsMessage(' + { + "type":"send", + "quotation":"_ALL", + "quotations":[ + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151}, + {"S":"TST","T":"2016/06/02 04:05:51","B":4} + ] + } +'); + +$client->addWsMessage(' + { + "type":"send", + "quotation":"BTCUSD", + "quotations":[ + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151} + ] + } +'); + +$client->addWsMessage(' + { + "type":"send", + "quotation":"BTCUSD", + "agg_period":"M1", + "agg_type":"AAPL", + "AAPL":[ + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151}, + {"S":"BTCUSD","T":"2016/06/11 04:05:51","B":568.151} + ] + } +'); + +$client->addWsMessage(['type' => 'conn_count']); +$client->addWsMessage(['type' => 'all', 'xx' => 123]); +$client->addWsMessage(['type' => 'test']); diff --git a/composer.json b/composer.json index a394b16..97b274a 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,9 @@ "clue/socket-raw": "~1.2", "pda/pheanstalk": "^3.1", "ruflin/Elastica": "^3.2", - "monolog/monolog": "^1.19" + "monolog/monolog": "^1.19", + "cboden/ratchet": "^0.3.5", + "ratchet/pawl": "^0.2.2" }, "require-dev": { "phpunit/phpunit": "4.7.*" @@ -30,10 +32,11 @@ }, "scripts": { "post-install-cmd": [ - "cp .env.example .env" + "cp .env.example .env", + "php command/setup/web_config_build.php" ], "post-update-cmd": [ - + "php command/setup/web_config_build.php" ] } } diff --git a/config/app.php b/config/app.php index cc6ab5e..5e3e67e 100644 --- a/config/app.php +++ b/config/app.php @@ -6,3 +6,12 @@ $dot_env->required('FINXLOG_QUOTATION_SERVER_ADDRESS')->notEmpty(); $dot_env->required('FINXLOG_QUOTATION_SERVER_PORT')->notEmpty(); +$dot_env->required('FINXLOG_ELASTICA_PARAM')->notEmpty(); + +if (getenv('FINXLOG_DEBUG')) { + require __DIR__ . '/dbg.php'; +} else { + require __DIR__ . '/pro.php'; +} + +assert(json_decode(getenv('FINXLOG_ELASTICA_PARAM'))); \ No newline at end of file diff --git a/config/dbg.php b/config/dbg.php new file mode 100644 index 0000000..48f38ec --- /dev/null +++ b/config/dbg.php @@ -0,0 +1,38 @@ +getStatus()->getIndexNames())); + + /** + * log message in elasticsearch + */ + \FinXLog\Module\Logger::log() + ->pushHandler( + new \Monolog\Handler\ElasticSearchHandler( + $logClient, + [], + Monolog\Logger::INFO + ) + ); +} catch (\Exception $e) { } \ No newline at end of file diff --git a/readme.md b/readme.md index eaeb26a..11e58d3 100644 --- a/readme.md +++ b/readme.md @@ -1,35 +1,38 @@ -сделано: +v 0.9.1 +Complete: - import - filter - save - -инструменты: - - база данных ElasticSearch для быстрого поиска и bigdata -опционально: - - менеджер очередей BeanstalkD(AMQP) (по умолчанию работает без него). причина: для быстрой доставки клиентам и выдерживания "любой" нагрузки без анализа дублей(bash-скрипт) - - composer - - monolog - - .env окружение - - -что надо сделать - - модуль для аггрегации данных https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html - - браузерный static интерфейс с highcharts, etc - - простое json API - -опционально и причина использования менеджера очередей: -параллельно с сохранением данных в БД, отправлять данные клиентам данные в реальном времени(поступления) через websocket или socket.io. -настройка преоретизации - сначала в бд, потом к клиентам - -можно: -опционально сохранение в SQL если "появятся" SQL задачи -использование ElasticSearch для всех логов - - + - queue + - elastic log + +Instruments: + - ElasticSearch for quick big data search + - Composer, PSR-4 + - Monolog + - ".ENV" environment + +Optional: + - BeanstalkD(AMQP Queue manager). reason for use: quick delivery for "any" load with single stream(bash-scrpit) + +@todo + - highcharts on websocket + - simple json API + - Ratchet + WAMP + ZMQ +Optional: + - sql db + - make quotation_exchange2db.sh + - lock for parallel import + # Install ```bash #install requirements -apt-get install php7.0 composer beanstalkd postgresql-9.5 php7.0-pgsql +sudo apt-get install php7.0 composer beanstalkd postgresql-9.5 php7.0-pgsql php7.0-mbstring + +#optional. cur. not ready +#sudo apt-get install php7.0-dev php-pear +#all questions - ENTER +#sudo pecl install event # https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-repositories.html wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - @@ -43,24 +46,44 @@ cd FinXLog composer update ``` - - +------------------------------- # Using Daemon for import quotation: ```bash -command/daemon/quotation_exchange2db.php +command/daemon/import/quotation_exchange2db.php ``` -## Optional: use with AMQP Queue for exchange high traffic +## Optional: import with AMQP for scaling high traffic important: direct import is more quickly, if server has free resource +important: direct import has minimal guarantee for stable: + mem leak + elastic can go to repair node with slow insert + crush the process leads to a loss of traffic +AMQP is depend by high performance, scalable beanstalk (and opensource client) + + +## Required(curently - ): WebSocket +WebSocket is require AMQP + or need implement async elasticsearch client with guzzle or reactphp/http-client + or need implement async reactphp/child-process +not ready but simple: + AMQP+AJAX+API can work without WebSocket +```bash +#load daemons can work on other servers with multiple fork +command/daemon/import/quotation_amqp2db.php +command/daemon/import/quotation_amqp2db.php fail +``` + + +## Default exchange IMPORT (not best choice) ```bash -command/daemon/quotation_exchange2amqp.php -command/daemon/quotation_amqp2db.php +#run only one daemons +command/daemon/import/quotation_exchange2amqp.php ``` -## Optional replacement for quotation_load.php +## Optional exchange IMPORT for hight traffic BASH replacement for quotation_load.php is direct linux-way socket to amqp pipe for high performance source: https://github.com/src-d/beanstool ### Install beanstool @@ -69,10 +92,7 @@ wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2. tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/ ``` -### Run -@todo: make auto-restart on network error +## run ```bash command/daemon/quotation_exchange2amqp.sh ``` - -@todo: make quotation_exchange2db.sh diff --git a/src/Classes/Exception/ConnectionError.php b/src/Classes/Exception/ConnectionError.php index 6d89d4c..9bc24df 100644 --- a/src/Classes/Exception/ConnectionError.php +++ b/src/Classes/Exception/ConnectionError.php @@ -3,7 +3,7 @@ use FinXLog\Iface; -class ConnectionError extends \Exception implements Iface\FinXLogException +class ConnectionError extends Error { } \ No newline at end of file diff --git a/src/Classes/Exception/Error.php b/src/Classes/Exception/Error.php index e30b21c..d3b854f 100644 --- a/src/Classes/Exception/Error.php +++ b/src/Classes/Exception/Error.php @@ -3,7 +3,7 @@ use FinXLog\Iface; -class Error extends \Exception implements Iface\FinXLogException +class Error extends \ErrorException implements Iface\FinXLogException { } \ No newline at end of file diff --git a/src/Classes/Exception/WrongImport.php b/src/Classes/Exception/WrongImport.php new file mode 100644 index 0000000..d42dbd4 --- /dev/null +++ b/src/Classes/Exception/WrongImport.php @@ -0,0 +1,9 @@ +params; + } + + public function setParams(array $params) + { + $this->params = $params; + + return $this; + } + public function addParams(array $params) + { + $this->params = $params + $this->params; + + return $this; + } +} \ No newline at end of file diff --git a/src/Classes/Iface/ElasticoConnector.php b/src/Classes/Iface/ElasticaConnector.php similarity index 90% rename from src/Classes/Iface/ElasticoConnector.php rename to src/Classes/Iface/ElasticaConnector.php index 9f97314..f24269b 100644 --- a/src/Classes/Iface/ElasticoConnector.php +++ b/src/Classes/Iface/ElasticaConnector.php @@ -1,7 +1,7 @@ getConnector()->getConnector(); } + /** + * @return \Elastica\Index + */ + public function getElasticIndex() + { + return $this->getDb() + ->getIndex($this->index); + } + + //@todo for simple query + //public function getResult(Query $query); + + /** + * @return \Elastica\ResultSet + */ + public function getResponse(\Elastica\Query $query) + { + return $this->getElasticIndex() + ->search($query); + } + public function checkConnector(Iface\Connector $connector) { assert($connector->getConnector() instanceof \Elastica\Client); @@ -96,4 +118,15 @@ public function checkConnector(Iface\Connector $connector) return $this; } + /** + * return + * @param Query $query + * @return \Elastica\Document[] + */ + public function getDocuments(Query $query) + { + return $this->getResponse($query) + ->getDocuments(); + } + } \ No newline at end of file diff --git a/src/Classes/Model/Quotation.php b/src/Classes/Model/Quotation.php index 947ef09..3891ac4 100644 --- a/src/Classes/Model/Quotation.php +++ b/src/Classes/Model/Quotation.php @@ -1,10 +1,44 @@ query_quotations, true); + if (!$quotation) { + $query['query']['bool']['must'] = []; + } else { + $query['query']['bool']['must'][0]['query_string']['query'] = $quotation; + } + + return $this->getDocuments(new Query($query)); + } } \ No newline at end of file diff --git a/src/Classes/Model/QuotationAgg.php b/src/Classes/Model/QuotationAgg.php new file mode 100644 index 0000000..71ed7c2 --- /dev/null +++ b/src/Classes/Model/QuotationAgg.php @@ -0,0 +1,163 @@ + '60', + 'M5' => '300', + 'H1' => '3600', + 'D1' => '86400', + 'W1' => '604800', + ]; + + /* + * more test, any first, not avg first + "first":{"top_hits":{"size": 1,"sort":[{"T": {"order": "asc"}}]}} + w/o deviation: extended_stats => stats + */ + private $query_agg_period = '{ + "query": { + "bool": { + "must": [ + { + "query_string": { + "default_field": "S", + "query": "BTCUSD" + } + } + ] + } + }, + "from": 0, + "size": 0, + "sort": [], + "aggs": { + "date": { + "date_histogram": { + "min_doc_count": 1, + "field": "T", + "interval": "1d", + "order": { + "_key": "desc" + } + }, + "aggs": { + "stat": { + "extended_stats": { + "field": "B", + "sigma": 3 + } + }, + "last": { + "terms": { + "size": 1, + "field": "T", + "order": { + "_term": "desc" + } + }, + "aggs": { + "avg": { + "avg": { + "field": "B" + } + } + } + }, + "first": { + "terms": { + "size": 1, + "field": "T", + "order": { + "_term": "asc" + } + }, + "aggs": { + "avg": { + "avg": { + "field": "B" + } + } + } + } + } + } + } + }'; + + /** + * @param string $subject exchange subject(EURUSD, USDBTC) + * @param string $interval (period) + * @return mixed + */ + public function getAgg($subject, $interval = 'day') + { + return $this->getAggregations( + $this->getDojiQuery( + $subject, + $interval + ) + ); + } + + /** + * AAPL (doji) historical OHLC data like the Google Finance API + * [date, open, high, low, close] + * @param $subject + * @param string $interval + * @return array + */ + public function getAAPL($subject, $interval = 'M1') + { + $prepared_result = []; + foreach ($this->getAgg($subject, $interval) as $agg) { + $prepared_result[] = [ + 1000 * strtotime($agg['key_as_string']), + (float) $agg['first']['buckets'][0]['avg']['value'], + (float) $agg['stat']['max'], + (float) $agg['stat']['min'], + (float) $agg['last']['buckets'][0]['avg']['value'], + ]; + } + + return $prepared_result; + } + + /** + * return + * @param Query $query + * @return array + */ + public function getAggregations(Query $query) + { + return current( //1st agg name is not important + $this->getResponse($query) + ->getAggregations() + )['buckets']; + } + + public function getDojiQuery($subject, $interval = 'day') + { + $query = json_decode($this->query_agg_period, true); + $query['query']['bool']['must'][0]['query_string']['query'] = $subject; + $query['aggs']['date']['date_histogram']['interval'] = + isset($this->agg_period[$interval]) + ? $this->agg_period[$interval] . 's' + : $interval; + + return new Query($query); + } + + public function getAggPeriod() + { + return $this->agg_period; + } +} diff --git a/src/Classes/Module/ClientQueue/AbsQuotation.php b/src/Classes/Module/ClientQueue/AbsQuotation.php new file mode 100644 index 0000000..09ef939 --- /dev/null +++ b/src/Classes/Module/ClientQueue/AbsQuotation.php @@ -0,0 +1,110 @@ +ws_connector = $ws_connector; + + return $this; + } + + /** + * @return Iface\WsConnector + * @throws WrongParams + */ + public function getWsConnector() + { + if (!$this->ws_connector) { + $this->ws_connector = $this->getDefaultWsConnector(); + } + + return $this->ws_connector; + } + + public function getDefaultWsConnector() + { + return (new Connector\RatchetClient) + ->setUrl($this->getWsUrl()); + } + + public function addWsMessage($message) + { + $this->getWsConnector()->send($message); + + return $this; + } + + public function getWsUrl() + { + if (empty(getenv('FINXLOG_WEBSOCKET_LISTEN_PORT'))) { + throw new WrongParams('empty: FINXLOG_WEBSOCKET_LISTEN_PORT'); + } + $host = ( + getenv('FINXLOG_WEBSOCKET_INTERNAL_HOST') + ? getenv('FINXLOG_WEBSOCKET_INTERNAL_HOST') + : ( + getenv('FINXLOG_WEBSOCKET_LISTEN_INTERFACE') + ? getenv('FINXLOG_WEBSOCKET_LISTEN_INTERFACE') + : '127.0.0.1' + ) + ); + + return "ws://$host" + . ( + getenv('FINXLOG_WEBSOCKET_LISTEN_PORT') + ? ':' . getenv('FINXLOG_WEBSOCKET_LISTEN_PORT') + : null + ) + . getenv('FINXLOG_WEBSOCKET_SERVICE_PATH'); + + } + + public function getModelQuotation() + { + if (!$this->model_quotation) { + $this->model_quotation = new Model\QuotationAgg(); + } + + return $this->model_quotation; + } + + public function getDefaultQueueConnector() + { + return new Connector\Queue( + getenv('FINXLOG_AMQP_TUBE_WS') + ); + } + + public function getFailQueueConnector() + { + //ignore + return $this; + } + public function failQueu(Job $queue_job) + { + //ignore + return $this; + } + + +} \ No newline at end of file diff --git a/src/Classes/Module/ClientQueue/SearchQuotation.php b/src/Classes/Module/ClientQueue/SearchQuotation.php new file mode 100644 index 0000000..b159693 --- /dev/null +++ b/src/Classes/Module/ClientQueue/SearchQuotation.php @@ -0,0 +1,88 @@ +getData(), true) + + [ + 'agg_type' => null, + 'agg_period' => null, + ]; + + if (!is_array($job) || empty($job['quotation'])) { + throw new Exception\WrongParams('!job with quotation'); + } + + switch ($job['agg_type']) { + case 'agg': + case 'full': + $quotations = $this->getModelQuotation() + ->getAgg($job['quotation'], $job['agg_period']); + break; + case 'doji': + case 'AAPL': + $quotations = $this->getModelQuotation() + ->getAAPL($job['quotation'], $job['agg_period']); + break; + default: + $quotations = $this->getModelQuotation() + ->getQuotations( + $job['quotation'] == static::QUOTATION_ALL + ? null + : $job['quotation'] + ); + foreach ($quotations as $key => $value) { + $quotations[$key] = $value->getData(); + } + break; + } + + $this->addWsMessage( + [ + 'type' => 'send', + 'quotations' => $quotations, + ] + + $job + ); + + return $this; + } + + public function run() + { + $this->getQueueConnector() + ->watch(); + //$this->getWsConnector()->send(['type' => 'test']); + + while (true) { + Logger::dbg(':amqp_wait:'); + $queue_job = $this->getQueueConnector()->reserve(); + Logger::dbg(':amqp_make:'); + try { + $this->makeJob($queue_job); + Logger::log()->debug('+'); + } catch (\Throwable $e) { + Logger::log()->debug('-'); + Logger::error("SaveQuotation exception: {$e->getMessage()}", ['e' => $e, 'job' => $queue_job]); + } + //try few times + $this->getQueueConnector() + ->delete($queue_job); + } + + return $this; + } +} \ No newline at end of file diff --git a/src/Classes/Module/Connector/Elastico.php b/src/Classes/Module/Connector/Elastica.php similarity index 67% rename from src/Classes/Module/Connector/Elastico.php rename to src/Classes/Module/Connector/Elastica.php index 95b051f..fc7643f 100644 --- a/src/Classes/Module/Connector/Elastico.php +++ b/src/Classes/Module/Connector/Elastica.php @@ -8,7 +8,7 @@ * Class Db * @package FinXLog\Module\Connector */ -class Elastico implements Iface\Connector +class Elastica implements Iface\Connector { use Traits\WithConnectorRaw; protected $param = []; @@ -22,11 +22,11 @@ public function setParam(array $params = []) public function getDefaultConnector(array $params = []) { - assert(!empty(getenv('FINXLOG_ELASTICO_PARAM'))); - assert(!empty(json_decode(getenv('FINXLOG_ELASTICO_PARAM')))); + assert(!empty(getenv('FINXLOG_ELASTICA_PARAM'))); + assert(!empty(json_decode(getenv('FINXLOG_ELASTICA_PARAM')))); return new \Elastica\Client( - $this->param + json_decode(getenv('FINXLOG_ELASTICO_PARAM'), true) + $this->param + json_decode(getenv('FINXLOG_ELASTICA_PARAM'), true) ); } } \ No newline at end of file diff --git a/src/Classes/Module/Connector/RatchetClient.php b/src/Classes/Module/Connector/RatchetClient.php new file mode 100644 index 0000000..effcce4 --- /dev/null +++ b/src/Classes/Module/Connector/RatchetClient.php @@ -0,0 +1,195 @@ + 'http://localhost' + ]; + private $subProtocols = []; + + /** + * BEFORE working with connector + * @param $url + */ + public function setUrl($url) + { + $this->url = $url; + + return $this; + } + + protected function getEventLoop() + { + if (!$this->event_loop) { + $this->event_loop = \React\EventLoop\Factory::create(); + } + + return $this->event_loop; + } + + protected function getPromise() + { + if (empty($this->url)) { + throw new WrongParams(__CLASS__ . 'use setUrl first'); + } + + if (!$this->promise) { + $connector = $this->getConnector(); + $this->promise = $connector( + $this->url, + $this->subProtocols, + $this->headers + ); + } + + return $this->promise; + } + + /** + * real work with $this->promise->then! + * @return \Ratchet\Client\Connector + */ + public function getDefaultConnector() + { + return new \Ratchet\Client\Connector($this->getEventLoop()); + } + + public function close() + { + if ($this->event_loop) { + $this->event_loop->stop(); + } + + return $this; + } + + public function reconnect() + { + + $this->event_loop = null; + $this->connector = null; + $this->promise = null; + Logger::dbg(':WS_RECONNECT:'); + + return $this; + } + + public function send($message, callable $incomingCallback = null) + { + $ratchet_module = $this; + + $is_first = empty($this->event_loop); + $event_loop = $this->getEventLoop(); + + $this->getPromise()->then( + function(\Ratchet\Client\WebSocket $conn) + use ($message, $event_loop, $incomingCallback, $ratchet_module) + { + $conn->on('error', function($error) use ($conn, $ratchet_module, $message, $incomingCallback) { + //@todo reconnect it's check only on second + //without send - 3 iterations + Logger::log()->notice("AMQP2WS(service): error"); + $ratchet_module->reconnect(); + if ($incomingCallback) { + $ratchet_module->send($message, $incomingCallback); + } else { + $ratchet_module->send($message); + } + }); + + if (!is_string($message)) { + $message = json_encode( + $message, + getenv('FINXLOG_DEBUG') + ? JSON_PRETTY_PRINT + : null + ); + } + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "AMQP2WS(service) send try:\t" . ( + getenv('FINXLOG_DEBUG') == \Monolog\Logger::DEBUG + ? preg_replace('~\s+~u', ' ', $message) . '!!!!!' + : substr($message, 0, 70) . '...' + ) + ); + if (empty($conn->listeners('close'))) { //new connect + $conn->on( + 'close', + function ($code = null, $reason = null) + use($incomingCallback) { + Logger::log()->info( + "AMQP2WS(service) closed: ({$code} - {$reason})" + ); + if ($incomingCallback) { + Logger::log()->info( + "AMQP2WS(service) incomingCallback: start" + ); + $incomingCallback(); + Logger::log()->info( + "AMQP2WS(service) incomingCallback: end" + ); + } + }); + //@todo check that the socket stream is empty(released) without on:event + $conn->on('message', function ($message) use ($conn) { + Logger::log()->warning( + "AMQP2WS(service) ignore message: {$message}" + ); + }); + } + } + $conn->send($message); + //exit from EventLoop + $event_loop->stop(); + + + + }, + function (\Throwable $e) + use ($event_loop, $message, $incomingCallback) + { + Logger::error( + "WS Service error: Could not connect: {$e->getMessage()}", + [ + 'e' => $e, + 'incomingCallback' => $incomingCallback, + ] + ); + $event_loop->stop(); + } + ); + if ($is_first) { + $event_loop->run(); + } + + //push all queue + $event_loop->tick(); + + return $this; + } + +} \ No newline at end of file diff --git a/src/Classes/Module/Import/SaveQuotation.php b/src/Classes/Module/Import/SaveQuotation.php deleted file mode 100644 index 8d7da80..0000000 --- a/src/Classes/Module/Import/SaveQuotation.php +++ /dev/null @@ -1,34 +0,0 @@ -getQueueConnector() - ->watch(); - - while ($limit === null || --$limit >= 0) { - $queu_job = $this->getQueueConnector()->reserve(); - try { - $this->importQuotation($queu_job->getData()); - $this->getQueueConnector() - ->delete($queu_job); - } catch (\Exception $e) { - $this->failQueu($queu_job); - } - } - - return $this; - } -} \ No newline at end of file diff --git a/src/Classes/Module/Import/Source/Telnet.php b/src/Classes/Module/Import/Source/Telnet.php deleted file mode 100644 index ff122c8..0000000 --- a/src/Classes/Module/Import/Source/Telnet.php +++ /dev/null @@ -1,33 +0,0 @@ -fail_queue_connector) { + if (!$this->fail_queue_connector) { $this->fail_queue_connector = new Connector\Queue( - getenv('FINXLOG_AMQP_TUBE_QUOTATION_FAIL') + getenv('FINXLOG_AMQP_TUBE_IMPORT_FAIL') ); } @@ -44,28 +42,36 @@ public function getModelQuotation() public function getDefaultQueueConnector() { return new Connector\Queue( - getenv('FINXLOG_AMQP_TUBE_QUOTATION') + getenv('FINXLOG_AMQP_TUBE_IMPORT') ); } public function failQueu(Job $queue_job) { - $this->getFailQueueConnector() - ->put( - $queue_job->getData() - ); + if ($this->getFailQueueConnector()) { + $this->getFailQueueConnector() + ->put( + $queue_job->getData() + ); + } - $this->getQueueConnector() - ->delete($queue_job); + if ($this->getQueueConnector()) { + $this->getQueueConnector() + ->delete($queue_job); + } + + return $this; } public function importQuotation($string) { $this->getModelQuotation() ->save( - Import\Source\Telnet::getFromRaw($string) + ImportQuotation\Source\Telnet::getFromRaw($string) ); + + return $this; } } \ No newline at end of file diff --git a/src/Classes/Module/Import/LoadQuotation.php b/src/Classes/Module/ImportQuotation/LoadQuotation.php similarity index 60% rename from src/Classes/Module/Import/LoadQuotation.php rename to src/Classes/Module/ImportQuotation/LoadQuotation.php index a996b84..50aa57f 100644 --- a/src/Classes/Module/Import/LoadQuotation.php +++ b/src/Classes/Module/ImportQuotation/LoadQuotation.php @@ -1,13 +1,11 @@ = 0) { - $this->saveJob( - $this->getConnector() - ->getQuotation() - ); - Logger::log()->info('job done'); + $import = null; + try { + $import = $this->getConnector() + ->getQuotation(); + } catch (\Throwable $e) { + Logger::log()->debug('-'); + Logger::error('LoadQuotation getQuotation error', $e); + } + + if ($import) { + try { + $this->addJob($import); + Logger::log()->debug('+'); + } catch (\Exception $e) { + Logger::log()->debug('-'); + Logger::error('LoadQuotation saveJob. lost: ' . var_export($import, true), $e); + } + } } return $this; } - public function saveJob($string) + public function addJob($string) { if ($this->work_with_amqp) { if ($this->getQueueConnector()) { diff --git a/src/Classes/Module/ImportQuotation/SaveQuotation.php b/src/Classes/Module/ImportQuotation/SaveQuotation.php new file mode 100644 index 0000000..048f682 --- /dev/null +++ b/src/Classes/Module/ImportQuotation/SaveQuotation.php @@ -0,0 +1,38 @@ += 0) { + $queue_job = $this->getQueueConnector()->reserve(); + try { + $this->importQuotation($queue_job->getData()); + Logger::log()->debug('+'); + $this->getQueueConnector()->delete($queue_job); + } catch (Exception\WrongImport $e) { + //Logger::log()->debug('-'); + Logger::error('SaveQuotation WrongImport', $e); + $this->getQueueConnector()->delete($queue_job); + } catch (\Throwable $e) { + //Logger::log()->debug('-'); + Logger::error('SaveQuotation exception', $e); + $this->failQueu($queue_job); + } + } + + return $this; + } +} \ No newline at end of file diff --git a/src/Classes/Module/Import/Source/AbsSource.php b/src/Classes/Module/ImportQuotation/Source/AbsSource.php similarity index 67% rename from src/Classes/Module/Import/Source/AbsSource.php rename to src/Classes/Module/ImportQuotation/Source/AbsSource.php index 37ff91a..d3ad7fa 100644 --- a/src/Classes/Module/Import/Source/AbsSource.php +++ b/src/Classes/Module/ImportQuotation/Source/AbsSource.php @@ -1,9 +1,11 @@ 'MSG', 'T' => '2001-01-01', @@ -25,19 +27,21 @@ public static function validate(array $result) $result ) ) { - throw new Exception\WrongParam( + throw (new Exception\WrongImport( 'import quotatin with empty param: ' . implode(', ', array_keys($diff)) - ); + )) + ->setParams(['quotation' =>$result]); } if (!strtotime($result['T'])) { - throw new Exception\WrongParam('is not a valid datetime: ' . $result['T']); + throw (new Exception\WrongParams('is not a valid datetime: ' . $result['T'])) + ->setParams(['T' => $result['T']]); } } public static function filter(array $result) { - if (getenv('FINXLOG_FILTER_OTHER')) { + if (getenv('FINXLOG_IMPORT_FILTER_OTHER')) { $result = array_diff_key( $result, static::getValidQuotation() @@ -49,9 +53,9 @@ public static function filter(array $result) public static function prepare(array $result) { - assert(!empty($result['T'])); - $result['T'] = static::getTime($result['T']); + $result['B'] = (float) $result['B']; + assert(!empty($result['B'])); return $result; } @@ -59,7 +63,8 @@ public static function getTime($string) { $timestamp = strtotime($string); if (!$timestamp) { - throw new Exception\WrongParam('is not a valid datetime: ' . $string); + throw (new Exception\WrongParams('is not a valid datetime: ' . $string)) + ->setParams(['T' => $string]); } return date(static::DATETIME_FORMAT, $timestamp); } diff --git a/src/Classes/Module/ImportQuotation/Source/Telnet.php b/src/Classes/Module/ImportQuotation/Source/Telnet.php new file mode 100644 index 0000000..1312b9a --- /dev/null +++ b/src/Classes/Module/ImportQuotation/Source/Telnet.php @@ -0,0 +1,41 @@ +addParams(['import' => $string]); + } + return $result; + } +} \ No newline at end of file diff --git a/src/Classes/Module/Logger.php b/src/Classes/Module/Logger.php index 905ceee..de8726c 100644 --- a/src/Classes/Module/Logger.php +++ b/src/Classes/Module/Logger.php @@ -1,5 +1,6 @@ getLogger(); } + public static function error($message, $context = []) + { + if ($context instanceof \Throwable) { + $context = [ + 'exception' => get_class($context), + 'trace' =>$context->getTraceAsString() + ]; + if ($context instanceof Iface\ExceptionWithParams) { + $context = [ + 'params' => $context->getParams() + ]; + } + } + + return static::log()->error($message, $context); + } + + + public static function dbg($message, $context = []) + { + if ($context instanceof \Throwable) { + $context = [ + 'exception' => get_class($context), + 'trace' =>$context->getTraceAsString() + ]; + if ($context instanceof Iface\ExceptionWithParams) { + $context = [ + 'params' => $context->getParams() + ]; + } + } + + return static::log()->debug($message, $context); + } + public function setLogger(\Psr\Log\LoggerInterface $logger) { $this->logger = $logger; @@ -37,23 +76,66 @@ public function setLogger(\Psr\Log\LoggerInterface $logger) return $this; } + private function getDefaultCliFormater() + { + return new Monolog\Formatter\LineFormatter("%message%\n"); + } + + private function getDefaultSingleLineFormater() + { + return new Monolog\Formatter\LineFormatter('%message%'); + } + private function getLoggerDefault() { - $logger = new Monolog\Logger('Language'); - $logger - ->pushHandler( - new Monolog\Handler\StreamHandler( - 'php://stderr', - Monolog\Logger::WARNING + $logger = (new Monolog\Logger('Language')); + + + if (!getenv('FINXLOG_DEBUG')) { + $logger + ->pushHandler( + new Monolog\Handler\StreamHandler( + 'php://stderr', + Monolog\Logger::WARNING + ) + ); + } elseif (getenv('FINXLOG_DEBUG') <= Monolog\Logger::DEBUG) { + $logger + ->pushHandler( + ( + new Monolog\Handler\StreamHandler( + 'php://stderr', + Monolog\Logger::INFO + ) + ) + ->setFormatter( + new Monolog\Formatter\LineFormatter("\n") + ) ) - ) - ->pushHandler( - new Monolog\Handler\StreamHandler( - 'php://stdout', - Monolog\Logger::INFO, - false + ->pushHandler( + ( + new Monolog\Handler\StreamHandler( + 'php://stderr', + Monolog\Logger::DEBUG + ) + ) + ->setFormatter( + $this->getDefaultSingleLineFormater() + ) + ); + } else { + $logger->pushHandler( + ( + new Monolog\Handler\StreamHandler( + 'php://stdout', + getenv('FINXLOG_DEBUG') > Monolog\Logger::DEBUG + ? getenv('FINXLOG_DEBUG') + : Monolog\Logger::INFO + ) ) + ->setFormatter($this->getDefaultCliFormater()) ); + } return $logger; } diff --git a/src/Classes/Module/Ratchet/QuotationServer.php b/src/Classes/Module/Ratchet/QuotationServer.php new file mode 100644 index 0000000..4d1ec67 --- /dev/null +++ b/src/Classes/Module/Ratchet/QuotationServer.php @@ -0,0 +1,47 @@ +io_server) { + assert(getenv('FINXLOG_WEBSOCKET_LISTEN_PORT') > 0); + + $this->io_server = IoServer::factory( + new HttpServer( + new WsServer( + new QuotationWebSocketDelivery() + ) + ), + getenv('FINXLOG_WEBSOCKET_LISTEN_PORT'), + getenv('FINXLOG_WEBSOCKET_LISTEN_INTERFACE') ?: '0.0.0.0' + ); + } + + return $this->io_server; + } + + public function setIoServer(IoServer $io_server) + { + $this->io_server = $io_server; + + return $this; + } + + public function run() + { + Logger::dbg(':ws_wait:'); + $this->getIoServer()->run(); + } +} diff --git a/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php new file mode 100644 index 0000000..03158fc --- /dev/null +++ b/src/Classes/Module/Ratchet/QuotationWebSocketDelivery.php @@ -0,0 +1,368 @@ +clients) { + $this->clients = new \SplObjectStorage; + } + + return $this->clients; + } + + /** + * is service => allow incoming + * @param ConnectionInterface|RFC6455\Connection $conn + * @return bool + */ + protected function isService(ConnectionInterface $conn, $msg = null) + { + if ( + $conn->resourceId + && isset($this->service_by_resource[$conn->resourceId]) + ) { + return $this->service_by_resource[$conn->resourceId]; + } + + $is_service = true; + + if (strlen(getenv('FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP'))) { + $is_service = $is_service && (bool) preg_match( + '~' . getenv('FINXLOG_WEBSOCKET_SERVICE_FILTER_ADDR_REGEXP') . '~iu', + $conn->remoteAddress + ); + } + + try { + if (getenv('FINXLOG_WEBSOCKET_SERVICE_PATH')) { + $is_service = ( + $is_service + && $conn instanceof RFC6455\Connection + && $conn->WebSocket->request + && false !== strstr( + $conn->WebSocket->request->getPath(), + getenv('FINXLOG_WEBSOCKET_SERVICE_PATH'), + 1 + ) + ); + } + } catch (\Throwable $e) { + Logger::error( + $e->getMessage(), + ['e' => $e] + ); + $is_service = false; + } + + return $this->service_by_resource[$conn->resourceId] = $is_service; + } + + /** + * @param ConnectionInterface|RFC6455\Connection $conn + */ + public function onOpen(ConnectionInterface $conn) + { + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "WS: New " + . ( + $this->isService($conn) + ? 'service' + : 'client' + ) + . " connection: {$conn->resourceId}", + [ + 'is_service' => $this->isService($conn), + 'conn' => $conn + ] + ); + } + } + + public function getPreparedMessage($msg, ConnectionInterface $conn = null) + { + $message = json_decode($msg, true); + if ( + empty($message) + || !is_array($message) + || empty($message['type']) + ) { + throw new WrongParams("WS: !msg[type]: " .var_export($msg, true)); + } + if ( + !empty($message['quotation']) + && !preg_match('~^[\w\d_\.]+$~u', $message['quotation']) + ) { + throw (new WrongParams("WS: wrong quotation: $msg")) + ->setParams(['quotation']); + } + if ($message['type'] == 'subscribe' && empty($message['quotation'])) { + $message['quotation'] = static::QUOTATION_ALL; + } + + if ( + !empty($message['agg_period']) + && !preg_match('~^[\w\d_\.]+$~u', $message['agg_period']) + ) { + throw (new WrongParams("WS: wrong agg_period: $msg")) + ->setParams(['agg_period']); + } + + return $message; + } + + /** + * @param ConnectionInterface|RFC6455\Connection $conn + */ + public function onMessage(ConnectionInterface $conn, $msg) + { + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "WS " . ($this->isService($conn) ? 'service' : 'client'). + " msg from #{$conn->resourceId}:" . mb_substr($msg, 0, 100) . '...', + [ + 'conn' => $conn, + 'is_service' => $this->isService($conn), + 'conn_count' => count($this->service_by_resource), + ] + ); + } + + if (!$this->isService($conn) && strlen($msg) > 100000) { + Logger::log()->warning( + "WS: msg with " . round(strlen($msg) / 1000). "kb", + [ + 'conn' => $conn, + 'is_service' => $this->isService($conn), + 'conn_count' => count($this->service_by_resource), + ] + ); + } + + try { + $message = $this->getPreparedMessage($msg, $conn); + if ($this->isService($conn, $message)) { + $this->addServiceIncoming($conn, $message); + } + //allow service subscribe + $this->addClientIncoming($conn, $message); + } catch (\Throwable $e) { + $error_message = ['type' => 'error']; + if (getenv('FINXLOG_DEBUG') && getenv('FINXLOG_DEBUG') < 300 /* \Monolog\Logger::NOTICE */) { + $error_message['error'] = [ + 'class' => get_class($e), + 'message' => $e->getMessage(), + ]; + } + $conn->send(json_encode($error_message)); + Logger::log()->warning( + "WS: {$e->getMessage()} from " + . ( + $this->isService($conn) + ? 'service' + : 'client' + ) + . " {$conn->resourceId}: : {$msg}", + [ + 'e' => $e, + 'conn' => $conn, + 'is_service' => $this->isService($conn), + 'conn_count' => count($this->service_by_resource), + ] + ); + } + } + /** + * delivery to clients + * @param ConnectionInterface|RFC6455\Connection $conn + * @param array $message + */ + protected function addServiceIncoming(ConnectionInterface $conn, array $message) + { + assert($this->isService($conn)); + assert(!empty($message['type'])); + + switch ($message['type']) { + case 'status': + case 'stat': + $conn->send(json_encode([ + 'conn_count' => count($this->service_by_resource), + ])); + break; + case 'all': + foreach ($this->getClients() as $client) { + if (!$this->isService($client)) { + $conn->send(json_encode($message)); + } + } + break; + case 'send': + foreach ( + $this->getSubscribers()->get($message) + as $subscribe + ) { + $subscribe['ws']->send(json_encode( + ['type' => 'subscribe'] + + $message + )); + } + Logger::log()->debug(':2js:'); + + break; + } + } + + /** + * @return Subscribers + */ + public function getSubscribers() + { + if (!$this->subscribers) { + $this->subscribers = new Subscribers; + } + + return $this->subscribers; + } + + protected function checkPeriod($period, $field_name = null) + { + if ( + strlen($period) + && !is_numeric($period) + && empty((new QuotationAgg)->getAggPeriod()[$period]) + ) { + $e = new WrongParams( + "WS: wrong period: $period" + . ($field_name !== null ? " field: $field_name" : '') + ); + if ($field_name !== null) { + $e->setParams($field_name); + } + + throw $e; + } + + return $this; + } + + /** + * set client opts + * @param ConnectionInterface|RFC6455\Connection $conn + * @param array $message + */ + protected function addClientIncoming(ConnectionInterface $conn, array $message) + { + if (empty($message['type'])) { + Logger::log()->warning('WS client? message without type'); + return $this; + } + + switch ($message['type']) { + case 'quotations': + $conn->send(json_encode([ + "type" => 'quotations', + 'quotations' => ['BTCUSD','USDBTC','USDEUR','EURUSD'] + ])); + break; + case 'subscribe': + if (!empty($message['agg_period'])) { + $this->checkPeriod($message['agg_period']); + } + $this->getSubscribers() + ->add(['ws' => $conn] + $message); + + //load previous period + $this->addJob($message); + Logger::log()->info('AMQP add:' . json_encode($message)); + break; + case 'unsubscribe': + $this->getSubscribers()->drop($conn, $message); + Logger::log()->info('AMQP unsubscribe'); + break; + } + + return $this; + } + + /** + * @param ConnectionInterface|RFC6455\Connection $conn + */ + public function onClose(ConnectionInterface $conn) + { + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "WS:Disconnect with #{$conn->resourceId}", + [ + 'conn' => $conn, + ] + ); + } + + unset($this->service_by_resource[$conn->resourceId]); + $this->getSubscribers()->drop($conn); + $this->getClients()->detach($conn); + } + + /** + * @param ConnectionInterface|RFC6455\Connection $conn + * @param \Exception $e + */ + public function onError(ConnectionInterface $conn, \Exception $e) + { + if (getenv('FINXLOG_DEBUG')) { + Logger::log()->info( + "WS: " . get_class($e) . ": {$e->getMessage()} with #{$conn->resourceId}", + [ + 'e' => $e, + 'conn' => $conn + ] + ); + } + $conn->close(); + } + + public function getDefaultQueueConnector() + { + return new Connector\Queue( + getenv('FINXLOG_AMQP_TUBE_WS') + ); + } + + public function addJob(array $array) + { + $this->getQueueConnector() + ->put($array); + + return $this; + } +} diff --git a/src/Classes/Module/Ratchet/Subscribers.php b/src/Classes/Module/Ratchet/Subscribers.php new file mode 100644 index 0000000..960a7e8 --- /dev/null +++ b/src/Classes/Module/Ratchet/Subscribers.php @@ -0,0 +1,106 @@ +getKeyByMessage($key); + } + + return empty($this->subscribers[$key]) + ? [] + : $this->subscribers[$key]; + } + + + /** + * @param RFC6455\Connection|ConnectionInterface $conn + * @return array + */ + public function getByConn(ConnectionInterface $conn) + { + if (empty($this->map_by_conn[$conn->resourceId])) { + return []; + } + $result = []; + foreach ($this->map_by_conn[$conn->resourceId] as $key) { + if (empty($this->subscribers[$key][$conn->resourceId])) { + Logger::error('subscribers: mapped key is missing'); + } else { + $result[] = $this->subscribers[$key][$conn->resourceId]; + } + } + + return $result; + } + + /** + * @param RFC6455\Connection|ConnectionInterface $conn + * @param string $key + * @return $this + */ + public function drop(ConnectionInterface $conn, $key = null) + { + if ($key) { + assert(isset($this->map_by_conn[$conn->resourceId][$key])); + assert(isset($this->subscribers[$key][$conn->resourceId])); + unset($this->map_by_conn[$conn->resourceId][$key]); + unset($this->subscribers[$key][$conn->resourceId]); + + if (empty($this->subscribers[$key])) { + unset($this->subscribers[$key]); + } + } elseif (isset($this->map_by_conn[$conn->resourceId])) { + foreach ((array) $this->map_by_conn[$conn->resourceId] as $key) { + unset($this->subscribers[$key][$conn->resourceId]); + if (empty($this->subscribers[$key])) { + unset($this->subscribers[$key]); + } + } + unset($this->map_by_conn[$conn->resourceId]); + } + + return $this; + } + + /** + * @param string|array $message + * @return $this + */ + public function add(array $message) + { + $key = $this->getKeyByMessage($message); + $this->subscribers[$key][$message['ws']->resourceId] = $message; + $this->map_by_conn[$message['ws']->resourceId] = $key; + + return $this; + } +} diff --git a/tests/ConnectorQuotationTest.php b/tests/ConnectorQuotationTest.php index 58b9188..3e08ff4 100644 --- a/tests/ConnectorQuotationTest.php +++ b/tests/ConnectorQuotationTest.php @@ -16,11 +16,12 @@ public function getApp() return $this->app; } - public function test_connector() + public function test_basic() { $this->assertTrue($this->getApp()->getDefaultConnector() instanceof \Socket\Raw\Socket); $this->assertTrue($this->getApp()->getConnector() instanceof \Socket\Raw\Socket); } + public function test_read() { $this->assertTrue(strlen($this->getApp()->read()) > 0); @@ -34,7 +35,7 @@ public function test_read() 'wrong result:' . $result ); - $quotation = \FinXLog\Module\Import\Source\Telnet::getFromRaw($result); + $quotation = \FinXLog\Module\ImportQuotation\Source\Telnet::getFromRaw($result); $this->assertTrue(is_array($quotation)); $this->assertTrue(count($quotation) == 3); $this->assertTrue($quotation === $quotation + ['T' => 1, 'B'=>2, 'S'=>3]); diff --git a/tests/ModelQuotationAggTest.php b/tests/ModelQuotationAggTest.php new file mode 100644 index 0000000..a9526f3 --- /dev/null +++ b/tests/ModelQuotationAggTest.php @@ -0,0 +1,42 @@ +app) { + $this->app = new \FinXLog\Model\QuotationAgg(); + } + + return $this->app; + } + + public function test_basic() + { + $this->assertTrue($this->getApp() instanceof \FinXLog\Model\AbsModel); + $this->assertTrue($this->getApp()->getIndex() == 'quotation'); + $this->assertTrue($this->getApp()->getType() == 'quotation'); + } + + public function test_query() + { + $this->assertTrue($this->getApp()->getDojiQuery('q') instanceof \Elastica\Query); + $query = $this->getApp()->getDojiQuery('q')->toArray(); + $this->assertTrue(is_array($query)); + $this->assertTrue(!empty($query['query']['bool']['must'][0]['query_string']['default_field'])); + + } + + public function test_getAgg() + { + try { + $this->assertTrue(is_array($this->getApp()->getAgg('BTCUSD'))); + } catch (\Elastica\Exception\ConnectionException $e) { + //connection error + } catch (Throwable $e) { + $this->assertTrue(false, '!getDoji'); + } + } +} diff --git a/tests/ModelQuotationTest.php b/tests/ModelQuotationTest.php new file mode 100644 index 0000000..a55bae0 --- /dev/null +++ b/tests/ModelQuotationTest.php @@ -0,0 +1,28 @@ +app) { + $this->app = new \FinXLog\Model\Quotation(); + } + + return $this->app; + } + + public function test_basic() + { + $this->assertTrue($this->getApp() instanceof \FinXLog\Model\AbsModel); + $this->assertTrue($this->getApp()->getDefaultConnector() instanceof \FinXLog\Module\Connector\Elastica); + $this->assertTrue($this->getApp()->getConnector() instanceof \FinXLog\Module\Connector\Elastica); + $this->assertTrue($this->getApp()->getDb() instanceof \Elastica\Client); + $this->assertTrue($this->getApp()->getElasticIndex() instanceof \Elastica\Index); + $this->assertTrue($this->getApp()->getPreparedDocument('string') instanceof \Elastica\Document); + + $this->assertTrue($this->getApp()->getIndex() == 'quotation'); + $this->assertTrue($this->getApp()->getType() == 'quotation'); + } +} diff --git a/www/index.html b/www/index.html new file mode 100644 index 0000000..7e3a07c --- /dev/null +++ b/www/index.html @@ -0,0 +1,15 @@ + +
+ + + + + + + + +@todo: + + + + diff --git a/www/js/.gitkeep b/www/js/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/www/js/app.js b/www/js/app.js new file mode 100644 index 0000000..67498a8 --- /dev/null +++ b/www/js/app.js @@ -0,0 +1,302 @@ +var FinXLog = { + //namespace + config: {}, + start: function() { + //this.graph.start(); + this.ws.start(); + this.ws.subscribe('BTCUSD', 'M1', 'AAPL'); + }, + dbg: function(msg, obj) { + if (!FinXLog.config.FINXLOG_DEBUG) return; + console.log(msg); + if (obj) console.log(obj); + }, + ws: { + conn: null, + subscribes: [ +// {type: 'subscribe', quotation: '_ALL', doji: null} +// {type: 'subscribe', quotation: 'BTCUSD'} +// {type: 'subscribe', quotation: 'BTCUSD', 'agg_type': 'AAPL', 'agg_period': 'H1'} + ], + query_onstart: [ + //prepare selector and build table + {type: 'quotations'} + ], + start: function (){ + this.conn = new WebSocket(this.getWsUrl()); + this.conn.onopen = function() { + var i; + if (FinXLog.config.FINXLOG_DEBUG) { + FinXLog.dbg('WS: open'); + } + for (i = 0; i < FinXLog.ws.query_onstart.length; ++i) { + FinXLog.ws.send(FinXLog.ws.query_onstart[i]); + } + for (i = 0; i < FinXLog.ws.subscribes.length; ++i) { + FinXLog.ws.send(FinXLog.ws.subscribes[i]); + } + FinXLog.ws.subscribe('BTCUSD', 'M1', 'AAPL'); + }; + + this.conn.onmessage = this.onmessage; + }, + /* + message2graph: function (message) { + var graph_data = {}; + for (var i = 0; i < message.quotations.length; ++i) { + var quotation = message.quotations[i]; + FinXLog.dbg(quotation); + if (typeof graph_data[quotation.S] == 'undefined') { + graph_data[quotation.S] = []; + } + graph_data[quotation.S].push({ + name: quotation.S, + x: new Date(quotation.T).getTime(), + y: quotation.B + }); + } + + return graph_data; + },*/ + onmessage: function (request) { + var message; + try { + message = JSON.parse(request.data); + FinXLog.dbg('incoming:'); + FinXLog.dbg(message); + } catch (err) { + FinXLog.dbg('wrong incoming:'); + FinXLog.dbg(err); + FinXLog.dbg(request.data); + throw err; + } + + switch (message.type) { + case 'quotations': { + $(message.quotations).each(function(i, item) { + var opt = document.createElement('option'); + opt.value = item; + opt.appendChild(document.createTextNode(item)) + $('select[name="quotation"]').append(opt) + }); + break; + } + case 'error': { + FinXLog.dbg(message); + + break; + } + case 'subscribe': { + switch(message.agg_type) { + case 'AAPL': { + FinXLog.draw(message.quotations); + } + default: { + FinXLog.dbg('bot ready draw for:', message); + } + } + break; + } + default: { + FinXLog.dbg(message); + } + } + }, + subscribe: function (subj, agg_period, agg_type, stop) { + var request = { + 'type': stop ? 'unsubscribe' : 'subscribe', + 'quotation': subj, + 'agg_period': agg_period, + 'agg_type': agg_type + }; + + this.send(request); + if (stop) { + for (var i = 0; i < this.subscribes.length; ++i) { + if (JSON.stringify(FinXLog.ws.subscribes[i]) == JSON.stringify(request)) { + //@todo + //FinXLog.ws.subscribes.remove + console.log('remove:'); + console.log(FinXLog.ws.subscribes[i]); + } + } + } else { + this.subscribes.push(request) + } + }, + send: function (obj) { + if (this.conn) { + FinXLog.dbg('send:', obj); + this.conn.send(JSON.stringify(obj)); + } else { + FinXLog.dbg('skip send(!conn):', obj); + } + }, + getWsUrl: function () + { + return 'ws://' + + ( + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_HOST + ? FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_HOST + : FinXLog.config.FINXLOG_DOMAIN + ) + + ( + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PORT + ? ':' + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PORT + : '' + ) + + FinXLog.config.FINXLOG_WEBSOCKET_EXTARNAL_PATH ; + } + }, + graph: { + list: [], + default_list: [ + { + quotation: 'EURUSD', + opts: {}, + conn: null + } + ], + start: function (){ + /*Highcharts.setOptions({ + global: { + useUTC: false + } + });*/ + for (var i = 0; i < FinXLog.graph.default_list.length; ++i) { + var graph_cur = FinXLog.graph.default_list[i]; + graph_cur.opts = this.blank_opts; + graph_cur.opts.title.text = 'Real Time ' + graph_cur.quotation; + graph_cur.opts.yAxis.title.text = 'ratio'; + //graph_cur.conn = new Highcharts.Chart(graph_cur.opts); + FinXLog.graph.list.push(graph_cur); + } + }, + blank_opts: { + title: {text: 'Real Time quotations'}, + xAxis: { + type: 'datetime', + tickPixelInterval: 100 + }, + yAxis: { + title: {text: 'Samples'}, + tickInterval: 10, + min: 0, + max: 100 + }, + tooltip: { + formatter: function () { + return '' + this.series.name + '