device_tests: allow custom runxfail

We can now selectively runxfail certain tests. This is useful for
accepting PRs into trezor-core:

1. trezor-core is going to get a pytest.ini that sets xfail_strict.
   That means that if an `xfail`ed test actually passes, that will
   break the test suite. So it will be visible when we implement
   a feature for which tests exist.
2. To allow PRs to pass the test suite without touching python-trezor
   directly, we add a new pytest.ini option: run_xfail.
   This adds a list of markers which will ignore `xfail`.
   So:
   2.1 First, the python-trezor PR marks the tests with the name
       of the feature. This commit already does that: Lisk tests
       are marked `@pytest.mark.lisk`, NEMs are `@pytest.mark.nem`,
       etc.
       The tests will be also marked with `xfail`, because the
       feature is not in core yet.
   2.2 Then, the trezor-core PR implements the feature, which makes
       the `xfail`ed tests pass. That breaks the test suite.
   2.3 To fix the test suite, the core PR also adds a `run_xfail`
       to `pytest.ini`: `run_xfail = lisk`.
       (it can take a list: `run_xfail = lisk nem stellar`)
       That will make the test suite behave as if the tests are not
       `xfail`ed. If the feature is implemented correctly, the tests
       will pass.
   2.4 When the PR is accepted to core, the next step should be
       a PR to python-trezor that removes the `xfail`s. After that,
       we should also remove the `run_xfail` option, just to be tidy.

trezorlib/tests/device_tests/conftest.py

@@ -22,13 +22,32 @@ except:
     TREZOR_VERSION = None
 
 
+def pytest_addoption(parser):
+    parser.addini("run_xfail", "List of markers that will run even if marked as xfail", "args", [])
+
+
 def pytest_runtest_setup(item):
-    '''
+    """
     Called for each test item (class, individual tests).
-    Ensures that 'skip_t2' tests are skipped on T2
+
-    and 'skip_t1' tests are skipped on T1.
+    Performs custom processing, mainly useful for trezor CI testing:
-    '''
+    * 'skip_t2' tests are skipped on T2 and 'skip_t1' tests are skipped on T1.
+    * no test should have both skips at the same time
+    * allows to 'runxfail' tests specified by 'run_xfail' in pytest.ini
+    """
+    if item.get_marker("skip_t1") and item.get_marker("skip_t2"):
+        pytest.fail("Don't skip tests for both trezors!")
+
     if item.get_marker("skip_t2") and TREZOR_VERSION == 2:
         pytest.skip("Test excluded on Trezor T")
     if item.get_marker("skip_t1") and TREZOR_VERSION == 1:
         pytest.skip("Test excluded on Trezor 1")
+
+    xfail = item.get_marker("xfail")
+    run_xfail = any(item.get_marker(marker) for marker in item.config.getini("run_xfail"))
+    if xfail and run_xfail:
+        # Deep hack: pytest's private _evalxfail helper determines whether the test should xfail or not.
+        # The helper caches its result even before this hook runs.
+        # Here we force-set the result to False, meaning "test does NOT xfail, run as normal"
+        # IOW, this is basically per-item "--runxfail"
+        item._evalxfail.result = False

trezorlib/tests/device_tests/test_msg_lisk_getaddress.py

@@ -21,6 +21,7 @@ import pytest
 from .common import TrezorTest
 
 
+@pytest.mark.lisk
 @pytest.mark.xfail  # drop when trezor-core PR #90 is merged
 @pytest.mark.skip_t1
 class TestMsgLiskGetaddress(TrezorTest):

trezorlib/tests/device_tests/test_msg_lisk_getpublickey.py

@@ -21,6 +21,7 @@ import pytest
 from .common import TrezorTest
 
 
+@pytest.mark.lisk
 @pytest.mark.xfail  # drop when trezor-core PR #90 is merged
 @pytest.mark.skip_t1
 class TestMsgLiskGetPublicKey(TrezorTest):

trezorlib/tests/device_tests/test_msg_lisk_signmessage.py

@@ -21,6 +21,7 @@ import pytest
 from .common import TrezorTest
 
 
+@pytest.mark.lisk
 @pytest.mark.xfail  # drop when trezor-core PR #90 is merged
 @pytest.mark.skip_t1
 class TestMsgLiskSignmessage(TrezorTest):

trezorlib/tests/device_tests/test_msg_lisk_signtx.py

@@ -25,6 +25,7 @@ from trezorlib import messages as proto
 PUBLIC_KEY = unhexlify('eb56d7bbb5e8ea9269405f7a8527fe126023d1db2c973cfac6f760b60ae27294')
 
 
+@pytest.mark.lisk
 @pytest.mark.xfail  # drop when trezor-core PR #90 is merged
 @pytest.mark.skip_t1
 class TestMsgLiskSignTx(TrezorTest):

trezorlib/tests/device_tests/test_msg_lisk_verifymessage.py

@@ -22,6 +22,7 @@ from .common import TrezorTest
 from trezorlib import messages as proto
 
 
+@pytest.mark.lisk
 @pytest.mark.xfail  # drop when trezor-core PR #90 is merged
 @pytest.mark.skip_t1
 class TestMsgLiskVerifymessage(TrezorTest):

trezorlib/tests/device_tests/test_msg_nem_getaddress.py

@@ -18,6 +18,7 @@
 from .common import *
 
 
+@pytest.mark.nem
 @pytest.mark.xfail  # to be removed when nem is merged
 class TestMsgNEMGetaddress(TrezorTest):
 

trezorlib/tests/device_tests/test_msg_nem_signtx_mosaics.py

@@ -20,6 +20,7 @@ from trezorlib import nem
 
 
 # assertion data from T1
+@pytest.mark.nem
 @pytest.mark.skip_t2
 class TestMsgNEMSignTxMosaics(TrezorTest):
 

trezorlib/tests/device_tests/test_msg_nem_signtx_mosaics_t2.py

@@ -23,6 +23,7 @@ import time
 
 
 # assertion data from T1
+@pytest.mark.nem
 @pytest.mark.skip_t1
 @pytest.mark.xfail  # to be removed when nem is merged
 class TestMsgNEMSignTxMosaics(TrezorTest):

trezorlib/tests/device_tests/test_msg_nem_signtx_multisig.py

@@ -21,6 +21,7 @@ from trezorlib import nem
 
 
 # assertion data from T1
+@pytest.mark.nem
 @pytest.mark.xfail  # to be removed when nem is merged
 class TestMsgNEMSignTxMultisig(TrezorTest):
 

trezorlib/tests/device_tests/test_msg_nem_signtx_others.py

@@ -21,6 +21,7 @@ from trezorlib import nem
 
 
 # assertion data from T1
+@pytest.mark.nem
 @pytest.mark.xfail  # to be removed when nem is merged
 class TestMsgNEMSignTxOther(TrezorTest):
 

trezorlib/tests/device_tests/test_msg_nem_signtx_transfers.py

@@ -22,6 +22,7 @@ from trezorlib import nem
 
 
 # assertion data from T1
+@pytest.mark.nem
 @pytest.mark.xfail  # to be removed when nem is merged
 class TestMsgNEMSignTx(TrezorTest):
 

trezorlib/tests/device_tests/test_msg_signtx_zcash.py

@@ -28,8 +28,11 @@ TxApiZcash = coins.tx_api["Zcash"]
 TXHASH_93373e = unhexlify('93373e63cc626c4a7d049ad775d6511bb5eba985f142db660c9b9f955c722f5c')
 
 
-@pytest.mark.skip_t1
+# Zcash reset their testnet, which broke our test because it was not properly cached.
-@pytest.mark.skip_t2
+# Then when we tried to revive it, Overwinter happened and now Trezor is incapable of
+# processing v3 transactions. So it's difficult to fix the test until we support v3.
+@pytest.mark.zcash
+@pytest.mark.xfail(reason="Zcash support is botched due to Overwinter")
 class TestMsgSigntxZcash(TrezorTest):
 
     def test_one_one_fee(self):

trezorlib/tests/device_tests/test_msg_stellar_get_public_key.py

@@ -19,6 +19,7 @@ from trezorlib import stellar
 import pytest
 
 
+@pytest.mark.stellar
 @pytest.mark.xfail(TREZOR_VERSION == 2, reason="T2 support is not yet finished")
 class TestMsgStellarGetPublicKey(TrezorTest):
 

trezorlib/tests/device_tests/test_msg_stellar_sign_transaction.py

@@ -25,6 +25,7 @@ from trezorlib import messages as proto
 import pytest
 
 
+@pytest.mark.stellar
 @pytest.mark.xfail(TREZOR_VERSION == 2, reason="T2 support is not yet finished")
 class TestMsgStellarSignTransaction(TrezorTest):