From d7bf007c731e20664fee8e87e043361b8f1df4c0 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sat, 18 Jul 2020 15:49:47 -0400 Subject: [PATCH 01/29] python: rdm string handling py3 compat picked from sdbbs pr#1615 9f30336fd70748f3d7907b07b579f6b2fddd0e67 --- python/ola/OlaClient.py | 6 +++++- python/ola/PidStore.py | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/python/ola/OlaClient.py b/python/ola/OlaClient.py index c1fb7935ca..6f60dace05 100644 --- a/python/ola/OlaClient.py +++ b/python/ola/OlaClient.py @@ -1357,7 +1357,11 @@ def _RDMMessage(self, universe, uid, sub_device, param_id, callback, data, request.uid.device_id = uid.device_id request.sub_device = sub_device request.param_id = param_id - request.data = data + if sys.version >= '3.2': + request.data = eval(data)[0] if data else bytes(data, 'utf-8') + else: + # eval(data)[0] if data else data - broke (MultiDim), 2019-10-31 + request.data = data # works, 2019-10-31 request.is_set = set request.include_raw_response = include_frames try: diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index 7c8a424b21..4734ad293a 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -458,7 +458,7 @@ def _AccountForMultiplierPack(self, value): raise ArgsValidationError( 'Conversion will lose data: %d -> %d' % (new_value, (new_value / multiplier * multiplier))) - new_value = new_value / multiplier + new_value = int(new_value / multiplier) else: try: @@ -645,7 +645,10 @@ def Pack(self, args): (self.name, self.min)) try: - data = struct.unpack('%ds' % arg_size, arg) + if sys.version >= '3.2': + data = struct.unpack('%ds' % arg_size, bytes(arg, 'utf8')) + else: + data = struct.unpack('%ds' % arg_size, arg) except struct.error as e: raise ArgsValidationError("Can't pack data: %s" % e) return data[0], 1 @@ -665,7 +668,10 @@ def Unpack(self, data): except struct.error as e: raise UnpackException(e) - return value[0].rstrip('\x00') + if sys.version >= '3.2': + return value[0].rstrip(bytes('\x00', 'utf-8')).decode('utf-8') + else: + return value[0].rstrip('\x00') def GetDescription(self, indent=0): indent = ' ' * indent @@ -807,7 +813,10 @@ def Pack(self, args): raise ArgsValidationError('Too many arguments, expected %d, got %d' % (arg_offset, len(args))) - return ''.join(data), arg_offset + if sys.version >= '3.2': + return ''.join(str(data)), arg_offset + else: + return ''.join(data), arg_offset elif self._group_size == 0: return '', 0 @@ -823,7 +832,7 @@ def Pack(self, args): if arg_offset < len(args): raise ArgsValidationError('Too many arguments, expected %d, got %d' % (arg_offset, len(args))) - return ''.join(data), arg_offset + return ''.join(str(data)), arg_offset def Unpack(self, data): """Unpack binary data. From cb60274029b2c9946b9d7b5ad39cb19a0e9fa93d Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sat, 18 Jul 2020 15:51:32 -0400 Subject: [PATCH 02/29] python: minor py3 print fix --- python/examples/ola_rdm_get.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/examples/ola_rdm_get.py b/python/examples/ola_rdm_get.py index 06a2542592..009ea7da8e 100755 --- a/python/examples/ola_rdm_get.py +++ b/python/examples/ola_rdm_get.py @@ -18,6 +18,7 @@ '''Get a PID from a UID.''' +from __future__ import print_function import cmd import getopt import os.path @@ -459,7 +460,7 @@ def main(): try: PidStore.GetStore(pid_location) except PidStore.MissingPLASAPIDs as e: - print e + print(e) sys.exit() controller = InteractiveModeController(universe, From 611365c3c475e49ecdb289fa0abcefc343dad42d Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Wed, 22 Jul 2020 15:49:59 -0400 Subject: [PATCH 03/29] python: additional py3 case from sdbbs and cleanup comment --- python/ola/OlaClient.py | 3 +-- python/ola/PidStore.py | 5 ++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/ola/OlaClient.py b/python/ola/OlaClient.py index 6f60dace05..3ca3f4ed5c 100644 --- a/python/ola/OlaClient.py +++ b/python/ola/OlaClient.py @@ -1360,8 +1360,7 @@ def _RDMMessage(self, universe, uid, sub_device, param_id, callback, data, if sys.version >= '3.2': request.data = eval(data)[0] if data else bytes(data, 'utf-8') else: - # eval(data)[0] if data else data - broke (MultiDim), 2019-10-31 - request.data = data # works, 2019-10-31 + request.data = data request.is_set = set request.include_raw_response = include_frames try: diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index 4734ad293a..78cd538f88 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -832,7 +832,10 @@ def Pack(self, args): if arg_offset < len(args): raise ArgsValidationError('Too many arguments, expected %d, got %d' % (arg_offset, len(args))) - return ''.join(str(data)), arg_offset + if sys.version >= '3.2': + return ''.join(str(data)), arg_offset + else: + return ''.join(data), arg_offset def Unpack(self, data): """Unpack binary data. From fe9a0a614a4781f681c6cb9ebde1b7cbfdb75b9d Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Thu, 23 Jul 2020 22:07:40 -0400 Subject: [PATCH 04/29] python: add compile test --- python/Makefile.mk | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/Makefile.mk b/python/Makefile.mk index 748c323bd7..88e40ef802 100644 --- a/python/Makefile.mk +++ b/python/Makefile.mk @@ -1,2 +1,17 @@ include python/examples/Makefile.mk include python/ola/Makefile.mk + +python/PyCompileTest.sh: python/Makefile.mk + mkdir -p $(top_builddir)/python +# restore this line when py3 compat is done for whole tree +# echo "$(PYTHON) -m compileall -f tools scripts python include data; exit \$$?" > $(top_builddir)/python/PyCompileTest.sh + echo "$(PYTHON) -m compileall -f python data; exit \$$?" > $(top_builddir)/python/PyCompileTest.sh + chmod +x $(top_builddir)/python/PyCompileTest.sh + +if BUILD_PYTHON_LIBS +test_scripts += \ + python/PyCompileTest.sh +endif + +CLEANFILES += \ + python/PyCompileTest.sh From d4acc280fc19068a7d79ee3a44c4c09144647f35 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Thu, 30 Jul 2020 17:23:11 -0400 Subject: [PATCH 05/29] python: add debug logging of messages --- python/ola/rpc/StreamRpcChannel.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ola/rpc/StreamRpcChannel.py b/python/ola/rpc/StreamRpcChannel.py index e414708be6..55b3313969 100644 --- a/python/ola/rpc/StreamRpcChannel.py +++ b/python/ola/rpc/StreamRpcChannel.py @@ -15,6 +15,7 @@ # StreamRpcChannel.py # Copyright (C) 2005 Simon Newton +import binascii import logging import struct from google.protobuf import service @@ -170,6 +171,7 @@ def _SendMessage(self, message): data = message.SerializeToString() # combine into one buffer to send so we avoid sending two packets data = self._EncodeHeader(len(data)) + data + logging.debug("send->" + str(binascii.hexlify(data))) sent_bytes = self._socket.send(data) if sent_bytes != len(data): @@ -240,6 +242,7 @@ def _ProcessIncomingData(self): if not raw_header: # not enough data yet return + logging.debug("recvhdr<-" + str(binascii.hexlify(raw_header))) header = struct.unpack('=L', raw_header)[0] version, size = self._DecodeHeader(header) @@ -254,6 +257,7 @@ def _ProcessIncomingData(self): # not enough data yet return + logging.debug("recvmsg<-" + str(binascii.hexlify(data))) if not self._skip_message: self._HandleNewMessage(data) self._expected_size = 0 From 721cf916c191196c4764fbc012a66fc98818c984 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Thu, 30 Jul 2020 17:24:32 -0400 Subject: [PATCH 06/29] python: clientwrappertest test with message check --- python/ola/ClientWrapperTest.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/ola/ClientWrapperTest.py b/python/ola/ClientWrapperTest.py index 8983bef4ff..0fb70af60d 100644 --- a/python/ola/ClientWrapperTest.py +++ b/python/ola/ClientWrapperTest.py @@ -17,6 +17,7 @@ # Copyright (C) 2019 Bruce Lowekamp import array +import binascii import datetime import socket # import timeout_decorator @@ -208,7 +209,13 @@ class results: def DataCallback(self): data = sockets[1].recv(4096) - self.assertTrue(len(data) > 100) + expected = binascii.unhexlify( + "7d000010080110001a0d557064617465446d784461746122680801126400000" + "000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000000000000000000000000000000000000000" + "000000") + self.assertEqual(data, expected) results.gotdata = True wrapper.AddEvent(0, wrapper.Stop) From bd19b889846dee8e777f65e1136c8efe5d741c35 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Thu, 30 Jul 2020 20:33:18 -0400 Subject: [PATCH 07/29] python: add RDMTest with mocked olad --- python/ola/Makefile.mk | 2 + python/ola/RDMTest.py | 100 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 python/ola/RDMTest.py diff --git a/python/ola/Makefile.mk b/python/ola/Makefile.mk index c671cdea66..d504bc8cfd 100644 --- a/python/ola/Makefile.mk +++ b/python/ola/Makefile.mk @@ -79,6 +79,7 @@ dist_check_SCRIPTS += \ python/ola/MACAddressTest.py \ python/ola/OlaClientTest.py \ python/ola/PidStoreTest.py \ + python/ola/RDMTest.py \ python/ola/TestUtils.py \ python/ola/UIDTest.py @@ -89,6 +90,7 @@ test_scripts += \ python/ola/MACAddressTest.py \ python/ola/OlaClientTest.sh \ python/ola/PidStoreTest.sh \ + python/ola/RDMTest.py \ python/ola/UIDTest.py endif diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py new file mode 100644 index 0000000000..f41bfda9e3 --- /dev/null +++ b/python/ola/RDMTest.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# ClientWrapperTest.py +# Copyright (C) 2019 Bruce Lowekamp + +import binascii +import socket +# import timeout_decorator +import unittest +from ola import PidStore +from ola.ClientWrapper import ClientWrapper +from ola.RDMAPI import RDMAPI +from ola.UID import UID + + +"""Test cases for RDM device commands.""" + +__author__ = 'bruce@lowekamp.net (Bruce Lowekamp)' + + +class RDMTest(unittest.TestCase): + # @timeout_decorator.timeout(2) + def testGetWithResponse(self): + """uses client to send an RDM get with mocked olad. + Regression test that confirms sent message is correct and + sends fixed response message.""" + sockets = socket.socketpair() + wrapper = ClientWrapper(sockets[0]) + pid_store = PidStore.GetStore() + client = wrapper.Client() + rdm_api = RDMAPI(client, pid_store) + + class results: + gotrequest = False + gotresponse = False + + def DataCallback(self): + # request and response for + # ola_rdm_get.py -u 1 --uid 7a70:ffffff00 device_info + # against olad dummy plugin + data = sockets[1].recv(4096) + expected = binascii.unhexlify( + "29000010080110001a0a52444d436f6d6d616e6422170801120908f0f4011500" + "ffffff180020602a0030003800") + self.assertEqual(data, expected) + results.gotrequest = True + response = binascii.unhexlify( + "3f0000100802100022390800100018002213010000017fff0000000300050204" + "00010000032860300038004a0908f0f4011500ffffff520908f0f40115ac1100" + "02580a") + sent_bytes = sockets[1].send(response) + self.assertEqual(sent_bytes, len(response)) + + def ResponseCallback(self, response, data, unpack_exception): + results.gotresponse = True + self.assertEqual(response.response_type, client.RDM_ACK) + self.assertEqual(response.pid, 0x60) + self.assertEqual(data["dmx_footprint"], 5) + self.assertEqual(data["software_version"], 3) + self.assertEqual(data["personality_count"], 4) + self.assertEqual(data["device_model"], 1) + self.assertEqual(data["current_personality"], 2) + self.assertEqual(data["protocol_major"], 1) + self.assertEqual(data["protocol_minor"], 0) + self.assertEqual(data["product_category"], 32767) + self.assertEqual(data["dmx_start_address"], 1) + self.assertEqual(data["sub_device_count"], 0) + self.assertEqual(data["sensor_count"], 3) + wrapper.AddEvent(0, wrapper.Stop) + + wrapper._ss.AddReadDescriptor(sockets[1], lambda: DataCallback(self)) + + uid = UID.FromString("7a70:ffffff00") + pid = pid_store.GetName("DEVICE_INFO", uid.manufacturer_id) + rdm_api.Get(1, uid, 0, pid, lambda x, y, z: ResponseCallback(self, x, y, z)) + + wrapper.Run() + + sockets[0].close() + sockets[1].close() + + self.assertTrue(results.gotrequest) + self.assertTrue(results.gotresponse) + + +if __name__ == '__main__': + unittest.main() From 4c63cd272581788b296a9ecbe77409e59a952fc1 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Thu, 30 Jul 2020 23:16:22 -0400 Subject: [PATCH 08/29] python: vpath fixes for RDMTest --- configure.ac | 1 + python/ola/Makefile.mk | 10 ++++++++-- python/ola/RDMTest.py | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 5624f834e6..d60c87e32b 100644 --- a/configure.ac +++ b/configure.ac @@ -955,6 +955,7 @@ AC_CONFIG_LINKS([python/ola/__init__.py:python/ola/__init__.py python/ola/MACAddress.py:python/ola/MACAddress.py python/ola/OlaClient.py:python/ola/OlaClient.py python/ola/PidStore.py:python/ola/PidStore.py + python/ola/RDMAPI.py:python/ola/RDMAPI.py python/ola/RDMConstants.py:python/ola/RDMConstants.py python/ola/TestUtils.py:python/ola/TestUtils.py python/ola/UID.py:python/ola/UID.py diff --git a/python/ola/Makefile.mk b/python/ola/Makefile.mk index d504bc8cfd..9552a53f06 100644 --- a/python/ola/Makefile.mk +++ b/python/ola/Makefile.mk @@ -73,6 +73,11 @@ python/ola/PidStoreTest.sh: python/ola/Makefile.mk echo "PYTHONPATH=${top_builddir}/python TESTDATADIR=$(srcdir)/common/rdm/testdata $(PYTHON) ${srcdir}/python/ola/PidStoreTest.py; exit \$$?" > $(top_builddir)/python/ola/PidStoreTest.sh chmod +x $(top_builddir)/python/ola/PidStoreTest.sh +python/ola/RDMTest.sh: python/ola/Makefile.mk + mkdir -p $(top_builddir)/python/ola + echo "PYTHONPATH=${top_builddir}/python PIDSTOREDIR=$(srcdir)/data/rdm $(PYTHON) ${srcdir}/python/ola/RDMTest.py; exit \$$?" > $(top_builddir)/python/ola/RDMTest.sh + chmod +x $(top_builddir)/python/ola/RDMTest.sh + dist_check_SCRIPTS += \ python/ola/DUBDecoderTest.py \ python/ola/ClientWrapperTest.py \ @@ -90,7 +95,7 @@ test_scripts += \ python/ola/MACAddressTest.py \ python/ola/OlaClientTest.sh \ python/ola/PidStoreTest.sh \ - python/ola/RDMTest.py \ + python/ola/RDMTest.sh \ python/ola/UIDTest.py endif @@ -98,4 +103,5 @@ CLEANFILES += \ python/ola/*.pyc \ python/ola/ClientWrapperTest.sh \ python/ola/OlaClientTest.sh \ - python/ola/PidStoreTest.sh + python/ola/PidStoreTest.sh \ + python/ola/RDMTest.sh diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index f41bfda9e3..1ba98cfafb 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -17,6 +17,7 @@ # Copyright (C) 2019 Bruce Lowekamp import binascii +import os import socket # import timeout_decorator import unittest @@ -30,6 +31,7 @@ __author__ = 'bruce@lowekamp.net (Bruce Lowekamp)' +global pidStorePath class RDMTest(unittest.TestCase): # @timeout_decorator.timeout(2) @@ -39,7 +41,7 @@ def testGetWithResponse(self): sends fixed response message.""" sockets = socket.socketpair() wrapper = ClientWrapper(sockets[0]) - pid_store = PidStore.GetStore() + pid_store = PidStore.GetStore(pidStorePath) client = wrapper.Client() rdm_api = RDMAPI(client, pid_store) @@ -97,4 +99,5 @@ def ResponseCallback(self, response, data, unpack_exception): if __name__ == '__main__': + pidStorePath = (os.environ.get('PIDSTOREDIR', "../data/rdm")) unittest.main() From 87041194ff97b4171942de20529e1fd6f5bc0c1f Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sat, 8 Aug 2020 15:41:16 -0400 Subject: [PATCH 09/29] remove config logging to find mac error with tests --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4880af5d83..366ddf592c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -353,8 +353,8 @@ before_install: after_failure: # Disabled as otherwise the logfile is too big # - if [ -f ${TRAVIS_BUILD_DIR}/config.log ]; then cat ${TRAVIS_BUILD_DIR}/config.log; fi - - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log; fi - - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log; fi +# - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log; fi +# - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log; fi From 3728c877fceb539726ccd0a4c25d4f69539c685d Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sat, 8 Aug 2020 15:48:22 -0400 Subject: [PATCH 10/29] python: minor fixes for rdmtest --- python/ola/RDMTest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index 1ba98cfafb..d4d8ae2887 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -13,7 +13,7 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # -# ClientWrapperTest.py +# RDMTest.py # Copyright (C) 2019 Bruce Lowekamp import binascii @@ -33,6 +33,7 @@ global pidStorePath + class RDMTest(unittest.TestCase): # @timeout_decorator.timeout(2) def testGetWithResponse(self): From eae77f7dd05e946cb09847f8a9b4b776b55b3318 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sat, 8 Aug 2020 16:19:22 -0400 Subject: [PATCH 11/29] python: switch linux clang build to python 3.8 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 366ddf592c..f7b921da59 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,7 +91,7 @@ matrix: dist: xenial compiler: clang env: TASK='compile' - python: '2.7' + python: '3.8' addons: apt: packages: From 2c06f6283b1810829a6dcea469286221d23113d6 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sat, 8 Aug 2020 17:25:21 -0400 Subject: [PATCH 12/29] python: do not recompile files that are up to date to avoid race conditions --- python/Makefile.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/Makefile.mk b/python/Makefile.mk index 88e40ef802..3ca641b6e0 100644 --- a/python/Makefile.mk +++ b/python/Makefile.mk @@ -5,7 +5,7 @@ python/PyCompileTest.sh: python/Makefile.mk mkdir -p $(top_builddir)/python # restore this line when py3 compat is done for whole tree # echo "$(PYTHON) -m compileall -f tools scripts python include data; exit \$$?" > $(top_builddir)/python/PyCompileTest.sh - echo "$(PYTHON) -m compileall -f python data; exit \$$?" > $(top_builddir)/python/PyCompileTest.sh + echo "$(PYTHON) -m compileall python data; exit \$$?" > $(top_builddir)/python/PyCompileTest.sh chmod +x $(top_builddir)/python/PyCompileTest.sh if BUILD_PYTHON_LIBS From 7a36c7fa11b7312282325962a35fe4b06e64608e Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sun, 8 Nov 2020 15:15:29 -0500 Subject: [PATCH 13/29] python: cleanup protocol regression test logging --- .gitignore | 2 ++ python/ola/ClientWrapperTest.py | 5 ++++- python/ola/PidStore.py | 2 +- python/ola/RDMTest.py | 5 ++++- python/ola/rpc/StreamRpcChannel.py | 14 ++++++++++---- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 7fe73e641e..16ac909d27 100644 --- a/.gitignore +++ b/.gitignore @@ -185,11 +185,13 @@ protoc/ola_protoc protoc/ola_protoc.exe protoc/ola_protoc_plugin protoc/ola_protoc_plugin.exe +python/PyCompileTest.sh python/examples/ola_rdm_set.py python/ola/ClientWrapperTest.sh python/ola/OlaClientTest.sh python/ola/PidStoreLocation.py python/ola/PidStoreTest.sh +python/ola/RDMTest.sh python/ola/Version.py python/ola/rpc/SimpleRpcControllerTest.sh slp/slp_client diff --git a/python/ola/ClientWrapperTest.py b/python/ola/ClientWrapperTest.py index 0fb70af60d..cd1e5cf2dc 100644 --- a/python/ola/ClientWrapperTest.py +++ b/python/ola/ClientWrapperTest.py @@ -215,7 +215,10 @@ def DataCallback(self): "000000000000000000000000000000000000000000000000000000000000000" "000000000000000000000000000000000000000000000000000000000000000" "000000") - self.assertEqual(data, expected) + self.assertEqual(data, expected, + msg="Regression check failed. If protocol change " + "was intended set expected to: " + + str(binascii.hexlify(data))) results.gotdata = True wrapper.AddEvent(0, wrapper.Stop) diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index 78cd538f88..b0f14204f8 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -53,7 +53,7 @@ class Error(Exception): class InvalidPidFormat(Error): - "Indicates the PID data file was invalid.""" + """Indicates the PID data file was invalid.""" class PidStructureException(Error): diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index d4d8ae2887..84e61d9b7e 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -58,7 +58,10 @@ def DataCallback(self): expected = binascii.unhexlify( "29000010080110001a0a52444d436f6d6d616e6422170801120908f0f4011500" "ffffff180020602a0030003800") - self.assertEqual(data, expected) + self.assertEqual(data, expected, + msg="Regression check failed. If protocol change " + "was intended set expected to: " + + str(binascii.hexlify(data))) results.gotrequest = True response = binascii.unhexlify( "3f0000100802100022390800100018002213010000017fff0000000300050204" diff --git a/python/ola/rpc/StreamRpcChannel.py b/python/ola/rpc/StreamRpcChannel.py index 55b3313969..67f67073a5 100644 --- a/python/ola/rpc/StreamRpcChannel.py +++ b/python/ola/rpc/StreamRpcChannel.py @@ -51,7 +51,8 @@ class StreamRpcChannel(service.RpcChannel): SIZE_MASK = 0x0fffffff RECEIVE_BUFFER_SIZE = 8192 - def __init__(self, socket, service_impl, close_callback=None): + def __init__(self, socket, service_impl, close_callback=None, + log_msgs=False): """Create a new StreamRpcChannel. Args: @@ -67,6 +68,7 @@ def __init__(self, socket, service_impl, close_callback=None): self._expected_size = None # The size of the message we're receiving self._skip_message = False # Skip the current message self._close_callback = close_callback + self._log_msgs = log_msgs # logs sent and rcvd messages for mocks def SocketReady(self): """Read data from the socket and handle when we get a full message. @@ -171,7 +173,9 @@ def _SendMessage(self, message): data = message.SerializeToString() # combine into one buffer to send so we avoid sending two packets data = self._EncodeHeader(len(data)) + data - logging.debug("send->" + str(binascii.hexlify(data))) + # this log is useful for building mock regression tests + if self._log_msgs: + logging.debug("send->" + str(binascii.hexlify(data))) sent_bytes = self._socket.send(data) if sent_bytes != len(data): @@ -242,7 +246,8 @@ def _ProcessIncomingData(self): if not raw_header: # not enough data yet return - logging.debug("recvhdr<-" + str(binascii.hexlify(raw_header))) + if self._log_msgs: + logging.debug("recvhdr<-" + str(binascii.hexlify(raw_header))) header = struct.unpack('=L', raw_header)[0] version, size = self._DecodeHeader(header) @@ -257,7 +262,8 @@ def _ProcessIncomingData(self): # not enough data yet return - logging.debug("recvmsg<-" + str(binascii.hexlify(data))) + if self._log_msgs: + logging.debug("recvmsg<-" + str(binascii.hexlify(data))) if not self._skip_message: self._HandleNewMessage(data) self._expected_size = 0 From b07fee1ee213d95c915b4efc540bd4304938c5ec Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Mon, 9 Nov 2020 10:19:44 -0500 Subject: [PATCH 14/29] python: refactor rdm pack/unpack with tests for 2/3 byte arrays --- python/ola/OlaClient.py | 11 ++--- python/ola/PidStore.py | 12 ++--- python/ola/PidStoreTest.py | 73 ++++++++++++++++++++++++++++++ python/ola/RDMTest.py | 58 +++++++++++++++++++++++- python/ola/rpc/StreamRpcChannel.py | 7 +-- 5 files changed, 141 insertions(+), 20 deletions(-) diff --git a/python/ola/OlaClient.py b/python/ola/OlaClient.py index 3ca3f4ed5c..8bfdd1b73a 100644 --- a/python/ola/OlaClient.py +++ b/python/ola/OlaClient.py @@ -1224,7 +1224,7 @@ def RunRDMDiscovery(self, universe, full, callback): raise OLADNotRunningException() return True - def RDMGet(self, universe, uid, sub_device, param_id, callback, data='', + def RDMGet(self, universe, uid, sub_device, param_id, callback, data=b'', include_frames=False): """Send an RDM get command. @@ -1246,7 +1246,7 @@ def RDMGet(self, universe, uid, sub_device, param_id, callback, data='', return self._RDMMessage(universe, uid, sub_device, param_id, callback, data, include_frames) - def RDMSet(self, universe, uid, sub_device, param_id, callback, data='', + def RDMSet(self, universe, uid, sub_device, param_id, callback, data=b'', include_frames=False): """Send an RDM set command. @@ -1274,7 +1274,7 @@ def SendRawRDMDiscovery(self, sub_device, param_id, callback, - data='', + data=b'', include_frames=False): """Send an RDM Discovery command. Unless you're writing RDM tests you shouldn't need to use this. @@ -1357,10 +1357,7 @@ def _RDMMessage(self, universe, uid, sub_device, param_id, callback, data, request.uid.device_id = uid.device_id request.sub_device = sub_device request.param_id = param_id - if sys.version >= '3.2': - request.data = eval(data)[0] if data else bytes(data, 'utf-8') - else: - request.data = data + request.data = data request.is_set = set request.include_raw_response = include_frames try: diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index b0f14204f8..b73e4bbecc 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -813,13 +813,10 @@ def Pack(self, args): raise ArgsValidationError('Too many arguments, expected %d, got %d' % (arg_offset, len(args))) - if sys.version >= '3.2': - return ''.join(str(data)), arg_offset - else: - return ''.join(data), arg_offset + return b''.join(data), arg_offset elif self._group_size == 0: - return '', 0 + return b'', 0 else: # this could be groups of fields, but we don't support that yet data = [] @@ -832,10 +829,7 @@ def Pack(self, args): if arg_offset < len(args): raise ArgsValidationError('Too many arguments, expected %d, got %d' % (arg_offset, len(args))) - if sys.version >= '3.2': - return ''.join(str(data)), arg_offset - else: - return ''.join(data), arg_offset + return b''.join(data), arg_offset def Unpack(self, data): """Unpack binary data. diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 5a6d92b0d3..85ae9dadec 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -198,6 +198,79 @@ def testCmp(self): self.assertNotEqual(hash(p1b), hash(p2)) self.assertNotEqual(hash(p1a), hash(p3)) + def testPackUnpack(self): + store = PidStore.PidStore() + store.Load([os.path.join(path, "test_pids.proto")]) + + pid = store.GetName("DMX_PERSONALITY_DESCRIPTION") + + # Pid.Pack only packs requests and Pid.Unpack responses + # so test in two halves + args = ["42"] + blob = pid.Pack(args, PidStore.RDM_GET) + decoded = pid._requests.get(PidStore.RDM_GET).Unpack(blob)[0] + self.assertEqual(decoded['personality'], 42) + + args = ["42", "7", "UnpackTest"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + decoded = pid.Unpack(blob, PidStore.RDM_GET) + self.assertEqual(decoded['personality'], 42) + self.assertEqual(decoded['slots_required'], 7) + self.assertEqual(decoded['name'], "UnpackTest") + + def testPackRanges(self): + store = PidStore.PidStore() + store.Load([os.path.join(path, "test_pids.proto")]) + + pid = store.GetName("REAL_TIME_CLOCK") + + args = ["2020", "6", "20", "20", "20", "20"] + blob = pid.Pack(args, PidStore.RDM_SET) + self.assertTrue(len(blob) > 1) + + with self.assertRaises(PidStore.ArgsValidationError): + args = ["2000", "6", "20", "20", "20", "20"] + blob = pid.Pack(args, PidStore.RDM_SET) + + with self.assertRaises(PidStore.ArgsValidationError): + args = ["2020", "0", "20", "20", "20", "20"] + blob = pid.Pack(args, PidStore.RDM_SET) + + with self.assertRaises(PidStore.ArgsValidationError): + args = ["2020", "13", "20", "20", "20", "20"] + blob = pid.Pack(args, PidStore.RDM_SET) + + with self.assertRaises(PidStore.ArgsValidationError): + args = ["2020", "255", "20", "20", "20", "20"] + blob = pid.Pack(args, PidStore.RDM_SET) + + with self.assertRaises(PidStore.ArgsValidationError): + args = ["2020", "-1", "20", "20", "20", "20"] + blob = pid.Pack(args, PidStore.RDM_SET) + + pid = store.GetName("LANGUAGE_CAPABILITIES") + args = ["Aa"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + self.assertTrue(len(blob) > 1) + + with self.assertRaises(PidStore.ArgsValidationError): + args = ["a"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + + with self.assertRaises(PidStore.ArgsValidationError): + args = ["zzz"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + + pid = store.GetName("STATUS_ID_DESCRIPTION") + args = [""] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + decoded = pid.Unpack(blob, PidStore.RDM_GET) + self.assertEqual(decoded['label'], "") + + with self.assertRaises(PidStore.ArgsValidationError): + args = ["123456789012345678901234567890123"] + blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + if __name__ == '__main__': path = (os.environ.get('TESTDATADIR', "../common/rdm/testdata")) diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index 84e61d9b7e..8e3d88da02 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -54,6 +54,7 @@ def DataCallback(self): # request and response for # ola_rdm_get.py -u 1 --uid 7a70:ffffff00 device_info # against olad dummy plugin + # enable logging in rpc/StreamRpcChannel.py data = sockets[1].recv(4096) expected = binascii.unhexlify( "29000010080110001a0a52444d436f6d6d616e6422170801120908f0f4011500" @@ -90,7 +91,7 @@ def ResponseCallback(self, response, data, unpack_exception): wrapper._ss.AddReadDescriptor(sockets[1], lambda: DataCallback(self)) uid = UID.FromString("7a70:ffffff00") - pid = pid_store.GetName("DEVICE_INFO", uid.manufacturer_id) + pid = pid_store.GetName("DEVICE_INFO") rdm_api.Get(1, uid, 0, pid, lambda x, y, z: ResponseCallback(self, x, y, z)) wrapper.Run() @@ -101,6 +102,61 @@ def ResponseCallback(self, response, data, unpack_exception): self.assertTrue(results.gotrequest) self.assertTrue(results.gotresponse) + # @timeout_decorator.timeout(2) + def testGetParamsWithResponse(self): + """uses client to send an RDM get with mocked olad. + Regression test that confirms sent message is correct and + sends fixed response message.""" + sockets = socket.socketpair() + wrapper = ClientWrapper(sockets[0]) + pid_store = PidStore.GetStore(pidStorePath) + client = wrapper.Client() + rdm_api = RDMAPI(client, pid_store) + + class results: + gotrequest = False + gotresponse = False + + def DataCallback(self): + # request and response for + # ola_rdm_get.py -u 1 --uid 7a70:ffffff00 parameter_description 17 + # against olad dummy plugin + # enable logging in rpc/StreamRpcChannel.py + data = sockets[1].recv(4096) + expected = binascii.unhexlify( + "2b000010080110001a0a52444d436f6d6d616e6422190801120908f0f4011500" + "ffffff180020512a02001130003800") + self.assertEqual(data, expected, + msg="Regression check failed. If protocol change " + "was intended set expected to: " + + str(binascii.hexlify(data))) + results.gotrequest = True + response = binascii.unhexlify( + "2e000010080210002228080010021800220200062851300038004a0908f0f401" + "1500ffffff520908f0f40115ac107de05811") + sent_bytes = sockets[1].send(response) + self.assertEqual(sent_bytes, len(response)) + + def ResponseCallback(self, response, data, unpack_exception): + results.gotresponse = True + self.assertEqual(response.response_type, client.RDM_NACK_REASON) + wrapper.AddEvent(0, wrapper.Stop) + + wrapper._ss.AddReadDescriptor(sockets[1], lambda: DataCallback(self)) + + uid = UID.FromString("7a70:ffffff00") + pid = pid_store.GetName("PARAMETER_DESCRIPTION") + rdm_api.Get(1, uid, 0, pid, + lambda x, y, z: ResponseCallback(self, x, y, z), args=["17"]) + + wrapper.Run() + + sockets[0].close() + sockets[1].close() + + self.assertTrue(results.gotrequest) + self.assertTrue(results.gotresponse) + if __name__ == '__main__': pidStorePath = (os.environ.get('PIDSTOREDIR', "../data/rdm")) diff --git a/python/ola/rpc/StreamRpcChannel.py b/python/ola/rpc/StreamRpcChannel.py index 67f67073a5..d4ddd9178b 100644 --- a/python/ola/rpc/StreamRpcChannel.py +++ b/python/ola/rpc/StreamRpcChannel.py @@ -51,8 +51,7 @@ class StreamRpcChannel(service.RpcChannel): SIZE_MASK = 0x0fffffff RECEIVE_BUFFER_SIZE = 8192 - def __init__(self, socket, service_impl, close_callback=None, - log_msgs=False): + def __init__(self, socket, service_impl, close_callback=None): """Create a new StreamRpcChannel. Args: @@ -68,7 +67,9 @@ def __init__(self, socket, service_impl, close_callback=None, self._expected_size = None # The size of the message we're receiving self._skip_message = False # Skip the current message self._close_callback = close_callback - self._log_msgs = log_msgs # logs sent and rcvd messages for mocks + self._log_msgs = False # set to enable wire message logging + if self._log_msgs: + logging.basicConfig(level=logging.DEBUG) def SocketReady(self): """Read data from the socket and handle when we get a full message. From 1d5a8802143e6ffa610c633a79c2570749b27df9 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Mon, 9 Nov 2020 15:20:34 -0500 Subject: [PATCH 15/29] python: clean up rdm tests a bit more --- python/ola/PidStore.py | 4 ++-- python/ola/PidStoreTest.py | 18 +++++++++++++----- python/ola/RDMTest.py | 19 ++++++++++++------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/python/ola/PidStore.py b/python/ola/PidStore.py index b73e4bbecc..bdf64e411b 100644 --- a/python/ola/PidStore.py +++ b/python/ola/PidStore.py @@ -669,9 +669,9 @@ def Unpack(self, data): raise UnpackException(e) if sys.version >= '3.2': - return value[0].rstrip(bytes('\x00', 'utf-8')).decode('utf-8') + return value[0].rstrip(b'\x00').decode('utf-8') else: - return value[0].rstrip('\x00') + return value[0].rstrip(b'\x00') def GetDescription(self, indent=0): indent = ' ' * indent diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 85ae9dadec..19e9144c2d 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -204,7 +204,7 @@ def testPackUnpack(self): pid = store.GetName("DMX_PERSONALITY_DESCRIPTION") - # Pid.Pack only packs requests and Pid.Unpack responses + # Pid.Pack only packs requests and Pid.Unpack only unpacks responses # so test in two halves args = ["42"] blob = pid.Pack(args, PidStore.RDM_GET) @@ -228,45 +228,53 @@ def testPackRanges(self): blob = pid.Pack(args, PidStore.RDM_SET) self.assertTrue(len(blob) > 1) + # invalid year (2002 < 2003) with self.assertRaises(PidStore.ArgsValidationError): - args = ["2000", "6", "20", "20", "20", "20"] + args = ["2002", "6", "20", "20", "20", "20"] blob = pid.Pack(args, PidStore.RDM_SET) + # invalid month < 1 with self.assertRaises(PidStore.ArgsValidationError): args = ["2020", "0", "20", "20", "20", "20"] blob = pid.Pack(args, PidStore.RDM_SET) + # invalid month > 12 with self.assertRaises(PidStore.ArgsValidationError): args = ["2020", "13", "20", "20", "20", "20"] blob = pid.Pack(args, PidStore.RDM_SET) + # invalid month > 255 with self.assertRaises(PidStore.ArgsValidationError): args = ["2020", "255", "20", "20", "20", "20"] blob = pid.Pack(args, PidStore.RDM_SET) + # invalid negative month with self.assertRaises(PidStore.ArgsValidationError): args = ["2020", "-1", "20", "20", "20", "20"] blob = pid.Pack(args, PidStore.RDM_SET) + # tests for string with min=max=2 pid = store.GetName("LANGUAGE_CAPABILITIES") - args = ["Aa"] + args = ["en"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] self.assertTrue(len(blob) > 1) with self.assertRaises(PidStore.ArgsValidationError): - args = ["a"] + args = ["e"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] with self.assertRaises(PidStore.ArgsValidationError): - args = ["zzz"] + args = ["enx"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + # valid empty string pid = store.GetName("STATUS_ID_DESCRIPTION") args = [""] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] decoded = pid.Unpack(blob, PidStore.RDM_GET) self.assertEqual(decoded['label'], "") + # string too long with self.assertRaises(PidStore.ArgsValidationError): args = ["123456789012345678901234567890123"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index 8e3d88da02..81349fad73 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -119,35 +119,40 @@ class results: def DataCallback(self): # request and response for - # ola_rdm_get.py -u 1 --uid 7a70:ffffff00 parameter_description 17 + # ola_rdm_get.py -u 1 --uid 7a70:ffffff00 DMX_PERSONALITY_DESCRIPTION 2 # against olad dummy plugin # enable logging in rpc/StreamRpcChannel.py data = sockets[1].recv(4096) expected = binascii.unhexlify( "2b000010080110001a0a52444d436f6d6d616e6422190801120908f0f4011500" - "ffffff180020512a02001130003800") + "ffffff180020e1012a010230003800") self.assertEqual(data, expected, msg="Regression check failed. If protocol change " "was intended set expected to: " + str(binascii.hexlify(data))) results.gotrequest = True response = binascii.unhexlify( - "2e000010080210002228080010021800220200062851300038004a0908f0f401" - "1500ffffff520908f0f40115ac107de05811") + "3d0000100802100022370800100018002210020005506572736f6e616c697479" + "203228e101300038004a0908f0f4011500ffffff520908f0f40115ac107de058" + "29" ) sent_bytes = sockets[1].send(response) self.assertEqual(sent_bytes, len(response)) def ResponseCallback(self, response, data, unpack_exception): results.gotresponse = True - self.assertEqual(response.response_type, client.RDM_NACK_REASON) + self.assertEqual(response.response_type, client.RDM_ACK) + self.assertEqual(response.pid, 0xe1) + self.assertEqual(data['personality'], 2) + self.assertEqual(data['slots_required'], 5) + self.assertEqual(data['name'], "Personality 2") wrapper.AddEvent(0, wrapper.Stop) wrapper._ss.AddReadDescriptor(sockets[1], lambda: DataCallback(self)) uid = UID.FromString("7a70:ffffff00") - pid = pid_store.GetName("PARAMETER_DESCRIPTION") + pid = pid_store.GetName("DMX_PERSONALITY_DESCRIPTION") rdm_api.Get(1, uid, 0, pid, - lambda x, y, z: ResponseCallback(self, x, y, z), args=["17"]) + lambda x, y, z: ResponseCallback(self, x, y, z), args=["2"]) wrapper.Run() From 17adb7730814b08c1661e048aa0f33ca58eafaa4 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Mon, 9 Nov 2020 16:21:54 -0500 Subject: [PATCH 16/29] python: experimental python3 travis build --- .travis.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 14e9e15e2b..1d62fe66c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -90,8 +90,9 @@ matrix: - os: linux dist: xenial compiler: clang - env: TASK='compile' - python: '3.8' + env: + - TASK='compile' + - PYTHON='python3' addons: apt: packages: @@ -104,7 +105,6 @@ matrix: dist: xenial compiler: gcc env: TASK='compile' - python: '2.7' addons: apt: packages: @@ -306,12 +306,12 @@ before_cache: install: # Match the version of protobuf being installed via apt -  - if [[ "$PROTOBUF" == "latest" ]]; then pip install --user protobuf; fi - - if [[ "$PROTOBUF" != "latest" ]]; then pip install --user protobuf==3.1.0; fi +  - if [[ "$PROTOBUF" == "latest" ]]; then pip install --user protobuf; pip3 install --user protobuf; fi + - if [[ "$PROTOBUF" != "latest" ]]; then pip install --user protobuf==3.1.0; pip3 install --user protobuf==3.1.0; fi # disable until can be added to all build variants - #- pip install --user timeout-decorator + #- pip install --user timeout-decorator; pip3 install --user timeout-decorator # We need to use pip rather than apt on Xenial - - if [ "$TRAVIS_OS_NAME" == "linux" ]; then pip install --user numpy; fi + - if [ "$TRAVIS_OS_NAME" == "linux" ]; then pip install --user numpy; pip3 install --user numpy; fi - if [ "$TASK" = "coverage" ]; then pip install --user cpp-coveralls; fi - if [ "$TASK" = "flake8" ]; then pip install --user flake8; fi - if [ "$TASK" = "codespell" ]; then pip3 install --user git+https://github.com/codespell-project/codespell.git; fi @@ -344,9 +344,10 @@ before_install: - if [ "$TRAVIS_OS_NAME" == "osx" -a "$CPPUNIT" == "1.14" ]; then brew install cppunit; fi # install the latest cppunit, which needs C++11 - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=/usr/local/opt/ccache/libexec:$PATH; fi # Use ccache on Mac too #Put back the old pip numpy we need to work - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip install --upgrade --no-deps --force-reinstall --user numpy; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip install --upgrade --no-deps --force-reinstall --user numpy; pip3 install --upgrade --no-deps --force-reinstall --user numpy; fi #Coverity doesn't work with g++ 5 or 6, so only upgrade to g++ 4.9 for that - if [ "$TRAVIS_OS_NAME" == "linux" -a \( "$TASK" = "compile" -o "$TASK" = "coverage" -o "$TASK" = "doxygen" \) -a "$CXX" = "g++" ]; then export CXX="ccache g++-9" CC="ccache gcc-9"; fi + - if [ "$PYTHON" == "python3" ]; then export PYTHON="python3"; fi - if [ "$TASK" = "coverity" -a "$CXX" = "g++" ]; then export CXX="g++-4.9" CC="gcc-4.9"; fi #Use the latest clang if we're compiling with clang - if [ "$TRAVIS_OS_NAME" == "linux" -a "$CXX" = "clang++" ]; then export CXX="clang++-6.0" CC="clang-6.0"; fi From c8dcc9253a85039c656bb9f038dd843c40a257aa Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Mon, 9 Nov 2020 18:08:40 -0500 Subject: [PATCH 17/29] python: extra space --- python/ola/RDMTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index 81349fad73..d64cab7f44 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -134,7 +134,7 @@ def DataCallback(self): response = binascii.unhexlify( "3d0000100802100022370800100018002210020005506572736f6e616c697479" "203228e101300038004a0908f0f4011500ffffff520908f0f40115ac107de058" - "29" ) + "29") sent_bytes = sockets[1].send(response) self.assertEqual(sent_bytes, len(response)) From df542e7d08d1cd75051c6c4f4b6c565336a1dbf5 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Mon, 9 Nov 2020 18:37:56 -0500 Subject: [PATCH 18/29] python: use pyenv to set py3 as global default --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1d62fe66c3..b6a8ac11f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -306,18 +306,19 @@ before_cache: install: # Match the version of protobuf being installed via apt -  - if [[ "$PROTOBUF" == "latest" ]]; then pip install --user protobuf; pip3 install --user protobuf; fi - - if [[ "$PROTOBUF" != "latest" ]]; then pip install --user protobuf==3.1.0; pip3 install --user protobuf==3.1.0; fi +  - if [[ "$PROTOBUF" == "latest" ]]; then pip install --user protobuf; fi + - if [[ "$PROTOBUF" != "latest" ]]; then pip install --user protobuf==3.1.0; fi # disable until can be added to all build variants - #- pip install --user timeout-decorator; pip3 install --user timeout-decorator + #- pip install --user timeout-decorator # We need to use pip rather than apt on Xenial - - if [ "$TRAVIS_OS_NAME" == "linux" ]; then pip install --user numpy; pip3 install --user numpy; fi + - if [ "$TRAVIS_OS_NAME" == "linux" ]; then pip install --user numpy; fi - if [ "$TASK" = "coverage" ]; then pip install --user cpp-coveralls; fi - if [ "$TASK" = "flake8" ]; then pip install --user flake8; fi - if [ "$TASK" = "codespell" ]; then pip3 install --user git+https://github.com/codespell-project/codespell.git; fi - if [ "$TASK" = "jshint" ]; then npm install -g grunt-cli; fi before_install: + - if [ "$PYTHON" == "python3" ]; then pyenv global 3.7.1 ; fi #Fix permissions for unbound (and possibly others) - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then if [ ! -d /usr/local/sbin ]; then sudo mkdir -p /usr/local/sbin && sudo chown -R $(whoami) /usr/local/sbin; fi; fi #Add a missing gnupg folder @@ -344,10 +345,9 @@ before_install: - if [ "$TRAVIS_OS_NAME" == "osx" -a "$CPPUNIT" == "1.14" ]; then brew install cppunit; fi # install the latest cppunit, which needs C++11 - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then PATH=/usr/local/opt/ccache/libexec:$PATH; fi # Use ccache on Mac too #Put back the old pip numpy we need to work - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip install --upgrade --no-deps --force-reinstall --user numpy; pip3 install --upgrade --no-deps --force-reinstall --user numpy; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip install --upgrade --no-deps --force-reinstall --user numpy; fi #Coverity doesn't work with g++ 5 or 6, so only upgrade to g++ 4.9 for that - if [ "$TRAVIS_OS_NAME" == "linux" -a \( "$TASK" = "compile" -o "$TASK" = "coverage" -o "$TASK" = "doxygen" \) -a "$CXX" = "g++" ]; then export CXX="ccache g++-9" CC="ccache gcc-9"; fi - - if [ "$PYTHON" == "python3" ]; then export PYTHON="python3"; fi - if [ "$TASK" = "coverity" -a "$CXX" = "g++" ]; then export CXX="g++-4.9" CC="gcc-4.9"; fi #Use the latest clang if we're compiling with clang - if [ "$TRAVIS_OS_NAME" == "linux" -a "$CXX" = "clang++" ]; then export CXX="clang++-6.0" CC="clang-6.0"; fi From de18f138c64e3b97329cf0fe869878be99e65b53 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Tue, 10 Nov 2020 06:48:52 -0500 Subject: [PATCH 19/29] python: remove __pycache__ pyc files on clean --- python/ola/Makefile.mk | 3 ++- python/ola/rpc/Makefile.mk | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/ola/Makefile.mk b/python/ola/Makefile.mk index 9552a53f06..e282ef187b 100644 --- a/python/ola/Makefile.mk +++ b/python/ola/Makefile.mk @@ -104,4 +104,5 @@ CLEANFILES += \ python/ola/ClientWrapperTest.sh \ python/ola/OlaClientTest.sh \ python/ola/PidStoreTest.sh \ - python/ola/RDMTest.sh + python/ola/RDMTest.sh \ + python/ola/__pycache__/* diff --git a/python/ola/rpc/Makefile.mk b/python/ola/rpc/Makefile.mk index 16e1e0354a..79d8443bf2 100644 --- a/python/ola/rpc/Makefile.mk +++ b/python/ola/rpc/Makefile.mk @@ -29,4 +29,5 @@ python/ola/rpc/SimpleRpcControllerTest.sh: python/ola/rpc/Makefile.mk chmod +x $(top_builddir)/python/ola/rpc/SimpleRpcControllerTest.sh CLEANFILES += python/ola/rpc/SimpleRpcControllerTest.sh \ - python/ola/rpc/*.pyc + python/ola/rpc/*.pyc \ + python/ola/rpc/__pycache__/* From 032ded55089673d7cea490753a3c1a6f8240f3b1 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Tue, 10 Nov 2020 19:45:38 -0500 Subject: [PATCH 20/29] python: fix test to match comment 256 --- python/ola/PidStoreTest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 19e9144c2d..8993011fd1 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -245,7 +245,7 @@ def testPackRanges(self): # invalid month > 255 with self.assertRaises(PidStore.ArgsValidationError): - args = ["2020", "255", "20", "20", "20", "20"] + args = ["2020", "256", "20", "20", "20", "20"] blob = pid.Pack(args, PidStore.RDM_SET) # invalid negative month From 95962ae53c8bd32700c0087afea1c3384bc3a0b7 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Tue, 10 Nov 2020 19:47:58 -0500 Subject: [PATCH 21/29] switch codespell build to use pyenv for py3 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6a8ac11f0..3196ab5bc9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -178,7 +178,7 @@ matrix: dist: xenial env: - TASK='codespell' - - PATH=/opt/python/3.7.1/bin:$PATH + - PYTHON='python3' addons: apt: packages: @@ -314,7 +314,7 @@ install: - if [ "$TRAVIS_OS_NAME" == "linux" ]; then pip install --user numpy; fi - if [ "$TASK" = "coverage" ]; then pip install --user cpp-coveralls; fi - if [ "$TASK" = "flake8" ]; then pip install --user flake8; fi - - if [ "$TASK" = "codespell" ]; then pip3 install --user git+https://github.com/codespell-project/codespell.git; fi + - if [ "$TASK" = "codespell" ]; then pip install --user git+https://github.com/codespell-project/codespell.git; fi - if [ "$TASK" = "jshint" ]; then npm install -g grunt-cli; fi before_install: From ea5ae311fa77abd18ceb7525273556c57b98ac36 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sat, 14 Nov 2020 21:17:23 -0500 Subject: [PATCH 22/29] set build log flags back to normal --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57efe3cc58..4f22ed78f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -358,8 +358,8 @@ before_install: after_failure: # Disabled as otherwise the logfile is too big # - if [ -f ${TRAVIS_BUILD_DIR}/config.log ]; then cat ${TRAVIS_BUILD_DIR}/config.log; fi -# - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log; fi -# - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log; fi + - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log; fi + - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log; fi From 206dd777515eda74af6daa00c8f51fdcd5b6b6b2 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sat, 14 Nov 2020 21:17:48 -0500 Subject: [PATCH 23/29] python: nack test and general cleanup --- python/ola/PidStoreTest.py | 9 +++- python/ola/RDMTest.py | 88 +++++++++++++++++++++++++++++++------- 2 files changed, 79 insertions(+), 18 deletions(-) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index 8993011fd1..cb21425144 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -226,7 +226,10 @@ def testPackRanges(self): args = ["2020", "6", "20", "20", "20", "20"] blob = pid.Pack(args, PidStore.RDM_SET) - self.assertTrue(len(blob) > 1) + decoded = pid._requests.get(PidStore.RDM_SET).Unpack(blob)[0] + self.assertEqual(decoded, {'year': 2020, 'month': 6, + 'day': 20, 'hour': 20, + 'minute': 20, 'second': 20}) # invalid year (2002 < 2003) with self.assertRaises(PidStore.ArgsValidationError): @@ -257,7 +260,8 @@ def testPackRanges(self): pid = store.GetName("LANGUAGE_CAPABILITIES") args = ["en"] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] - self.assertTrue(len(blob) > 1) + decoded = pid.Unpack(blob, PidStore.RDM_GET) + self.assertEqual(decoded, {'languages': [{'language': 'en'}]}) with self.assertRaises(PidStore.ArgsValidationError): args = ["e"] @@ -271,6 +275,7 @@ def testPackRanges(self): pid = store.GetName("STATUS_ID_DESCRIPTION") args = [""] blob = pid._responses.get(PidStore.RDM_GET).Pack(args)[0] + self.assertEqual(len(blob), 0) decoded = pid.Unpack(blob, PidStore.RDM_GET) self.assertEqual(decoded['label'], "") diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index d64cab7f44..3d175f3917 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -31,7 +31,7 @@ __author__ = 'bruce@lowekamp.net (Bruce Lowekamp)' -global pidStorePath +global pid_store_path class RDMTest(unittest.TestCase): @@ -42,13 +42,13 @@ def testGetWithResponse(self): sends fixed response message.""" sockets = socket.socketpair() wrapper = ClientWrapper(sockets[0]) - pid_store = PidStore.GetStore(pidStorePath) + pid_store = PidStore.GetStore(pid_store_path) client = wrapper.Client() rdm_api = RDMAPI(client, pid_store) class results: - gotrequest = False - gotresponse = False + got_request = False + got_response = False def DataCallback(self): # request and response for @@ -63,7 +63,7 @@ def DataCallback(self): msg="Regression check failed. If protocol change " "was intended set expected to: " + str(binascii.hexlify(data))) - results.gotrequest = True + results.got_request = True response = binascii.unhexlify( "3f0000100802100022390800100018002213010000017fff0000000300050204" "00010000032860300038004a0908f0f4011500ffffff520908f0f40115ac1100" @@ -72,7 +72,7 @@ def DataCallback(self): self.assertEqual(sent_bytes, len(response)) def ResponseCallback(self, response, data, unpack_exception): - results.gotresponse = True + results.got_response = True self.assertEqual(response.response_type, client.RDM_ACK) self.assertEqual(response.pid, 0x60) self.assertEqual(data["dmx_footprint"], 5) @@ -99,8 +99,8 @@ def ResponseCallback(self, response, data, unpack_exception): sockets[0].close() sockets[1].close() - self.assertTrue(results.gotrequest) - self.assertTrue(results.gotresponse) + self.assertTrue(results.got_request) + self.assertTrue(results.got_response) # @timeout_decorator.timeout(2) def testGetParamsWithResponse(self): @@ -109,13 +109,13 @@ def testGetParamsWithResponse(self): sends fixed response message.""" sockets = socket.socketpair() wrapper = ClientWrapper(sockets[0]) - pid_store = PidStore.GetStore(pidStorePath) + pid_store = PidStore.GetStore(pid_store_path) client = wrapper.Client() rdm_api = RDMAPI(client, pid_store) class results: - gotrequest = False - gotresponse = False + got_request = False + got_response = False def DataCallback(self): # request and response for @@ -130,7 +130,7 @@ def DataCallback(self): msg="Regression check failed. If protocol change " "was intended set expected to: " + str(binascii.hexlify(data))) - results.gotrequest = True + results.got_request = True response = binascii.unhexlify( "3d0000100802100022370800100018002210020005506572736f6e616c697479" "203228e101300038004a0908f0f4011500ffffff520908f0f40115ac107de058" @@ -139,7 +139,7 @@ def DataCallback(self): self.assertEqual(sent_bytes, len(response)) def ResponseCallback(self, response, data, unpack_exception): - results.gotresponse = True + results.got_response = True self.assertEqual(response.response_type, client.RDM_ACK) self.assertEqual(response.pid, 0xe1) self.assertEqual(data['personality'], 2) @@ -159,10 +159,66 @@ def ResponseCallback(self, response, data, unpack_exception): sockets[0].close() sockets[1].close() - self.assertTrue(results.gotrequest) - self.assertTrue(results.gotresponse) + self.assertTrue(results.got_request) + self.assertTrue(results.got_response) + + # @timeout_decorator.timeout(2) + def testSetParamsWithNack(self): + """uses client to send an RDM set with mocked olad. + Regression test that confirms sent message is correct and + sends fixed response message.""" + sockets = socket.socketpair() + wrapper = ClientWrapper(sockets[0]) + pid_store = PidStore.GetStore(pid_store_path) + client = wrapper.Client() + rdm_api = RDMAPI(client, pid_store) + + class results: + got_request = False + got_response = False + + def DataCallback(self): + # request and response for + # ola_rdm_set.py -u 1 --uid 7a70:ffffff00 DMX_PERSONALITY 10 + # against olad dummy plugin + # enable logging in rpc/StreamRpcChannel.py + data = sockets[1].recv(4096) + expected = binascii.unhexlify( + "2b000010080110001a0a52444d436f6d6d616e6422190801120908f0f401150" + "0ffffff180020e0012a010a30013800") + self.assertEqual(data, expected, + msg="Regression check failed. If protocol change " + "was intended set expected to: " + + str(binascii.hexlify(data))) + results.got_request = True + response = binascii.unhexlify( + "2f0000100802100022290800100218002202000628e001300138004a0908f0f" + "4011500ffffff520908f0f40115ac107de05831") + sent_bytes = sockets[1].send(response) + self.assertEqual(sent_bytes, len(response)) + + def ResponseCallback(self, response, data, unpack_exception): + results.got_response = True + self.assertEqual(response.response_type, client.RDM_NACK_REASON) + self.assertEqual(response.pid, 0xe0) + wrapper.AddEvent(0, wrapper.Stop) + + wrapper._ss.AddReadDescriptor(sockets[1], lambda: DataCallback(self)) + + uid = UID.FromString("7a70:ffffff00") + pid = pid_store.GetName("DMX_PERSONALITY") + rdm_api.Set(1, uid, 0, pid, + lambda x, y, z: ResponseCallback(self, x, y, z), args=["10"]) + + wrapper.Run() + + sockets[0].close() + sockets[1].close() + + self.assertTrue(results.got_request) + self.assertTrue(results.got_response) if __name__ == '__main__': - pidStorePath = (os.environ.get('PIDSTOREDIR', "../data/rdm")) + pid_store_path = (os.environ.get('PIDSTOREDIR', "../data/rdm")) unittest.main() From f178c6300c5c3fdb801161aabe57df6e409e07dd Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sun, 15 Nov 2020 07:38:44 -0500 Subject: [PATCH 24/29] change travis logs to always output test-suite.log --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4f22ed78f7..3167046cab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -356,12 +356,12 @@ before_install: - if [ "$TASK" == "spellintian" -o "$TASK" == "spellintian-duplicates" ]; then wget "http://old-releases.ubuntu.com/ubuntu/pool/main/l/lintian/lintian_2.5.104_all.deb"; sudo dpkg -i lintian_*.deb; sudo apt-get install -f -y; fi # Install a later lintian after_failure: + - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log; fi + - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log; fi # Disabled as otherwise the logfile is too big # - if [ -f ${TRAVIS_BUILD_DIR}/config.log ]; then cat ${TRAVIS_BUILD_DIR}/config.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log; fi - - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log; fi - - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log; fi after_success: - if [ "$TASK" = "coverage" ]; then coveralls --gcov /usr/bin/gcov-8 -b . -E '.*Test\.cpp$' -E '.*\.pb\.cc$' -E '.*\.pb\.cpp$' -E '.*\.pb\.h$' -E '.*\.yy\.cpp$' -E '.*\.tab\.cpp$' -E '.*\.tab\.h$' -E '.*/doxygen/examples.*$' --gcov-options '\-lp' > /dev/null; fi From 03a141e39ddf76de612bfb76e030962c3cc21bbf Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sun, 15 Nov 2020 12:43:46 -0500 Subject: [PATCH 25/29] python: add make variable for opting into python compile test --- Makefile.am | 4 ++++ data/Makefile.mk | 2 ++ include/Makefile.mk | 4 ++++ python/Makefile.mk | 6 +++--- scripts/Makefile.mk | 2 ++ tools/Makefile.mk | 3 +++ 6 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 scripts/Makefile.mk diff --git a/Makefile.am b/Makefile.am index c03a99070e..c741ff0b66 100644 --- a/Makefile.am +++ b/Makefile.am @@ -149,6 +149,9 @@ built_sources = # Test scripts are run if BUILD_TESTS is true. test_scripts = +# directories with python code that should be test compiled during the build +PYTHON_BUILD_DIRS = + # The includes # ----------------------------------------------------------------------------- @@ -173,6 +176,7 @@ include olad/Makefile.mk include protoc/Makefile.mk include python/Makefile.mk include tools/Makefile.mk +include scripts/Makefile.mk # ----------------------------------------------------------------------------- diff --git a/data/Makefile.mk b/data/Makefile.mk index cc6afb63f3..fae7e38891 100644 --- a/data/Makefile.mk +++ b/data/Makefile.mk @@ -1 +1,3 @@ include data/rdm/Makefile.mk + +PYTHON_BUILD_DIRS += data diff --git a/include/Makefile.mk b/include/Makefile.mk index c42383c800..08797660e1 100644 --- a/include/Makefile.mk +++ b/include/Makefile.mk @@ -1,2 +1,6 @@ include include/ola/Makefile.mk include include/olad/Makefile.mk + +# uncomment when include is py3 compatible +# PYTHON_BUILD_DIRS += include + diff --git a/python/Makefile.mk b/python/Makefile.mk index 3ca641b6e0..4436c891b3 100644 --- a/python/Makefile.mk +++ b/python/Makefile.mk @@ -3,11 +3,11 @@ include python/ola/Makefile.mk python/PyCompileTest.sh: python/Makefile.mk mkdir -p $(top_builddir)/python -# restore this line when py3 compat is done for whole tree -# echo "$(PYTHON) -m compileall -f tools scripts python include data; exit \$$?" > $(top_builddir)/python/PyCompileTest.sh - echo "$(PYTHON) -m compileall python data; exit \$$?" > $(top_builddir)/python/PyCompileTest.sh + echo "$(PYTHON) -m compileall -f $(PYTHON_BUILD_DIRS); exit \$$?" > $(top_builddir)/python/PyCompileTest.sh chmod +x $(top_builddir)/python/PyCompileTest.sh +PYTHON_BUILD_DIRS += python + if BUILD_PYTHON_LIBS test_scripts += \ python/PyCompileTest.sh diff --git a/scripts/Makefile.mk b/scripts/Makefile.mk new file mode 100644 index 0000000000..199f0f9e58 --- /dev/null +++ b/scripts/Makefile.mk @@ -0,0 +1,2 @@ +# uncomment when scripts is py3 compatible +# PYTHON_BUILD_DIRS += scripts diff --git a/tools/Makefile.mk b/tools/Makefile.mk index 4a30fa4dfc..afa5742817 100644 --- a/tools/Makefile.mk +++ b/tools/Makefile.mk @@ -13,3 +13,6 @@ dist_noinst_DATA += \ tools/ola_mon/index.html \ tools/ola_mon/ola_mon.conf \ tools/ola_mon/ola_mon.py + +# uncomment when tools is py3 compatible +# PYTHON_BUILD_DIRS += tools From f636029cd5f8fb00e8a866f5288e20a1150b74bf Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sun, 15 Nov 2020 14:59:50 -0500 Subject: [PATCH 26/29] random build fixes --- .travis.yml | 3 +-- Makefile.am | 2 +- python/Makefile.mk | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3167046cab..c82317ef11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -358,8 +358,7 @@ before_install: after_failure: - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/test-suite.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/test-suite.log; fi -# Disabled as otherwise the logfile is too big -# - if [ -f ${TRAVIS_BUILD_DIR}/config.log ]; then cat ${TRAVIS_BUILD_DIR}/config.log; fi + - if [ -f ${TRAVIS_BUILD_DIR}/config.log ]; then cat ${TRAVIS_BUILD_DIR}/config.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/config.log; fi - if [ -f ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log ]; then cat ${TRAVIS_BUILD_DIR}/ola-*/_build/sub/config.log; fi diff --git a/Makefile.am b/Makefile.am index c741ff0b66..130bbb001a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -175,8 +175,8 @@ include plugins/Makefile.mk include olad/Makefile.mk include protoc/Makefile.mk include python/Makefile.mk -include tools/Makefile.mk include scripts/Makefile.mk +include tools/Makefile.mk # ----------------------------------------------------------------------------- diff --git a/python/Makefile.mk b/python/Makefile.mk index 4436c891b3..1682171414 100644 --- a/python/Makefile.mk +++ b/python/Makefile.mk @@ -3,7 +3,7 @@ include python/ola/Makefile.mk python/PyCompileTest.sh: python/Makefile.mk mkdir -p $(top_builddir)/python - echo "$(PYTHON) -m compileall -f $(PYTHON_BUILD_DIRS); exit \$$?" > $(top_builddir)/python/PyCompileTest.sh + echo "$(PYTHON) -m compileall $(PYTHON_BUILD_DIRS); exit \$$?" > $(top_builddir)/python/PyCompileTest.sh chmod +x $(top_builddir)/python/PyCompileTest.sh PYTHON_BUILD_DIRS += python From c4ce2f86e566f0dab3865256b5f48fab232d8dba Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Sun, 15 Nov 2020 15:00:07 -0500 Subject: [PATCH 27/29] python: check specific type of nack in response --- python/ola/RDMTest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index 3d175f3917..a57d9634b6 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -23,6 +23,7 @@ import unittest from ola import PidStore from ola.ClientWrapper import ClientWrapper +from ola.OlaClient import RDMNack from ola.RDMAPI import RDMAPI from ola.UID import UID @@ -201,6 +202,9 @@ def ResponseCallback(self, response, data, unpack_exception): results.got_response = True self.assertEqual(response.response_type, client.RDM_NACK_REASON) self.assertEqual(response.pid, 0xe0) + self.assertEqual(response.nack_reason, + RDMNack.LookupCode(RDMNack.NACK_SYMBOLS_TO_VALUES + ['NR_DATA_OUT_OF_RANGE'][0])) wrapper.AddEvent(0, wrapper.Stop) wrapper._ss.AddReadDescriptor(sockets[1], lambda: DataCallback(self)) From 6d6fed7e6ff40d4fa950b253d6ca5452c24bca04 Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Mon, 16 Nov 2020 15:29:19 -0500 Subject: [PATCH 28/29] python: simpler RDMNack syntax and add comment --- python/ola/OlaClient.py | 6 ++++++ python/ola/RDMTest.py | 4 +--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/ola/OlaClient.py b/python/ola/OlaClient.py index 8bfdd1b73a..20856bdfc1 100644 --- a/python/ola/OlaClient.py +++ b/python/ola/OlaClient.py @@ -429,6 +429,12 @@ def message(self): class RDMNack(object): + """Nack response to a request. + + Individual NACK response reasons can be access as attrs, e.g. + RMDNack.NR_FORMAT_ERROR + """ + NACK_SYMBOLS_TO_VALUES = { 'NR_UNKNOWN_PID': (0, 'Unknown PID'), 'NR_FORMAT_ERROR': (1, 'Format Error'), diff --git a/python/ola/RDMTest.py b/python/ola/RDMTest.py index a57d9634b6..773612124b 100644 --- a/python/ola/RDMTest.py +++ b/python/ola/RDMTest.py @@ -202,9 +202,7 @@ def ResponseCallback(self, response, data, unpack_exception): results.got_response = True self.assertEqual(response.response_type, client.RDM_NACK_REASON) self.assertEqual(response.pid, 0xe0) - self.assertEqual(response.nack_reason, - RDMNack.LookupCode(RDMNack.NACK_SYMBOLS_TO_VALUES - ['NR_DATA_OUT_OF_RANGE'][0])) + self.assertEqual(response.nack_reason, RDMNack.NR_DATA_OUT_OF_RANGE) wrapper.AddEvent(0, wrapper.Stop) wrapper._ss.AddReadDescriptor(sockets[1], lambda: DataCallback(self)) From 1eff5f27b229e9fb96102dcbae29487bb3b69c0c Mon Sep 17 00:00:00 2001 From: Bruce Lowekamp Date: Mon, 16 Nov 2020 15:42:05 -0500 Subject: [PATCH 29/29] python: add encoding check of RTC and comments --- python/ola/PidStoreTest.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/ola/PidStoreTest.py b/python/ola/PidStoreTest.py index cb21425144..8f93c82d29 100755 --- a/python/ola/PidStoreTest.py +++ b/python/ola/PidStoreTest.py @@ -16,6 +16,7 @@ # PidStoreTest.py # Copyright (C) 2020 Bruce Lowekamp +import binascii import os import unittest import ola.PidStore as PidStore @@ -224,13 +225,16 @@ def testPackRanges(self): pid = store.GetName("REAL_TIME_CLOCK") - args = ["2020", "6", "20", "20", "20", "20"] + # first check encoding of valid RTC data + args = ["2020", "6", "20", "21", "22", "23"] blob = pid.Pack(args, PidStore.RDM_SET) + self.assertEqual(blob, binascii.unhexlify("07e40614151617")) decoded = pid._requests.get(PidStore.RDM_SET).Unpack(blob)[0] self.assertEqual(decoded, {'year': 2020, 'month': 6, - 'day': 20, 'hour': 20, - 'minute': 20, 'second': 20}) + 'day': 20, 'hour': 21, + 'minute': 22, 'second': 23}) + # next check that ranges are being enforced properly # invalid year (2002 < 2003) with self.assertRaises(PidStore.ArgsValidationError): args = ["2002", "6", "20", "20", "20", "20"]