From 6de1a96d3863a40d90cd3355f69a1342b78f59cf Mon Sep 17 00:00:00 2001 From: r0ny123 <49360849+r0ny123@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:37:32 +0000 Subject: [PATCH 1/3] Ask the server for a sample's jobs and download family and sample lists after Convert The job list for a sample sent every job of the server and filtered them in the plugin. The sample id now goes to the server as the job filter, and the exact test stays because the filter matches the id as text and older servers ignore sample_ids. Against mcrit.malpedia.io this takes 1.45 s down to 0.9 s. Convert waited for the family and sample lists. They now download in the background after the report is built, side by side and once for every widget that asks, which takes 3.9 s down to 2.4 s. The family chooser for an unknown sample opens once they have arrived. --- CHANGELOG.md | 6 +++ mcrit_plugin/core/McritInterface.py | 36 ++++++++++++--- mcrit_plugin/ui_qt/McritSession.py | 5 +-- .../ui_qt/widgets/BlockMatchWidget.py | 6 +-- .../ui_qt/widgets/FunctionMatchWidget.py | 6 +-- mcrit_plugin/ui_qt/widgets/MainWidget.py | 44 +++++++++++++------ tests/core/test_mcrit_interface.py | 43 ++++++++++++++++++ 7 files changed, 112 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd1676e..3c0ba5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,12 @@ Ninja applies to that plugin only. - Binary Ninja: MCRIT requests show in the status bar as background tasks named after what they do, such as "MCRIT: fetching labels for 151306 matched functions", like Binary Ninja's own analysis tasks. The plugin's log messages go to the log of the file they concern. +- Choosing a matching job asks the server for the jobs that mention the sample, instead of + downloading every job of the server and filtering them in the plugin. +- Convert no longer waits for the family and sample lists: they download in the background after + the report is built, side by side instead of one after the other, and one download serves every + widget that needs them. The family chooser for + an unknown sample opens once they are here. - The bundled MCRIT client has the interface of MCRIT 1.12.0's client: `sample_group_only` only on the cross compare, the raising error modes, a default connect timeout, the job selectors of `getQueueData` and the unique-block parameters. The plugin sets its own timeout, so requests diff --git a/mcrit_plugin/core/McritInterface.py b/mcrit_plugin/core/McritInterface.py index 9d065ad..c29cb8a 100644 --- a/mcrit_plugin/core/McritInterface.py +++ b/mcrit_plugin/core/McritInterface.py @@ -25,6 +25,7 @@ def __init__(self, parent, backend): self.backend = backend self.config = parent.config self._cache_lock = threading.Lock() + self._remote_info_lock = threading.Lock() self._mcrit_server = self.config.MCRIT_SERVER self.mcrit_client = McritClient(self.config.MCRIT_SERVER) timeout_value = self.config.MCRIT_REQUEST_TIMEOUT @@ -190,8 +191,12 @@ def queryJobs(self, sample_id=None): else: self.parent.local_widget.updateActivityInfo("Querying jobs.") try: - # fetch jobs - jobs = self.mcrit_client.getQueueData(filter="Matches") + # fetch jobs; the server narrows them by the sample id as text, which also matches + # other numbers, and older servers do not select by sample_ids, so the exact test + # below stays + jobs = self.mcrit_client.getQueueData( + filter="Matches" if sample_id is None else str(sample_id) + ) if jobs is None: self.parent.local_widget.updateActivityInfo("Job query failed.") return None @@ -200,10 +205,13 @@ def queryJobs(self, sample_id=None): jobs = [ job for job in jobs - if "(" + str(sample_id) + ")" in job.parameters - or "(" + str(sample_id) + "," in job.parameters - or "," + str(sample_id) + "," in job.parameters - or "," + str(sample_id) + ")" in job.parameters + if "Matches" in job.parameters + and ( + "(" + str(sample_id) + ")" in job.parameters + or "(" + str(sample_id) + "," in job.parameters + or "," + str(sample_id) + "," in job.parameters + or "," + str(sample_id) + ")" in job.parameters + ) ] if jobs: self.parent.local_widget.updateActivityInfo("Success! Fetched Jobs.") @@ -247,6 +255,22 @@ def getMatchingJobById(self, job_id): except Exception as exc: self._reportFailure("Result query", exc) + def ensureRemoteInformation(self): + """Download the family and sample lists unless the session has them. Callable from several + threads: one downloads while the others wait for it instead of repeating it.""" + with self._remote_info_lock: + # the two lists are independent, so they download side by side + downloads = [] + if self.parent.family_infos is None: + downloads.append(threading.Thread(target=self.queryAllFamilyEntries, daemon=True)) + if self.parent.sample_infos is None: + downloads.append(threading.Thread(target=self.queryAllSampleEntries, daemon=True)) + for download in downloads: + download.start() + for download in downloads: + download.join() + return self.parent.family_infos is not None and self.parent.sample_infos is not None + def queryAllFamilyEntries(self): self.parent.local_widget.updateActivityInfo("Querying for FamilyEntries") try: diff --git a/mcrit_plugin/ui_qt/McritSession.py b/mcrit_plugin/ui_qt/McritSession.py index bcd9291..542b59d 100644 --- a/mcrit_plugin/ui_qt/McritSession.py +++ b/mcrit_plugin/ui_qt/McritSession.py @@ -82,10 +82,7 @@ def getRemoteSampleInformation(self): """Download family/sample meta data; callable off the UI thread, so it touches no widget.""" time_before = time.time() print("[/] starting download of meta data from MCRIT...") - self.mcrit_interface.queryAllFamilyEntries() - print("[|] downloaded FamilyEntries!") - self.mcrit_interface.queryAllSampleEntries() - print("[|] downloaded SampleEntries!") + self.mcrit_interface.ensureRemoteInformation() print("[\\] this took %3.2f seconds.\n" % (time.time() - time_before)) def setupWidgets(self): diff --git a/mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py b/mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py index 5bc2a81..087c260 100644 --- a/mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py +++ b/mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py @@ -139,11 +139,7 @@ def _get_family_entry(self, family_id): def _fetch_remote_cache(self): """Request the family and sample lists if they are missing; touches no widget.""" - if self.parent.family_infos is None: - self.parent.mcrit_interface.queryAllFamilyEntries() - if self.parent.sample_infos is None: - self.parent.mcrit_interface.queryAllSampleEntries() - return self.parent.family_infos is not None and self.parent.sample_infos is not None + return self.parent.mcrit_interface.ensureRemoteInformation() def _show_remote_cache_unavailable(self): self.clearTable() diff --git a/mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py b/mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py index 8423b09..d943b33 100644 --- a/mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py +++ b/mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py @@ -137,11 +137,7 @@ def _get_family_entry(self, family_id): def _fetch_remote_cache(self): """Request the family and sample lists if they are missing; touches no widget.""" - if self.parent.family_infos is None: - self.parent.mcrit_interface.queryAllFamilyEntries() - if self.parent.sample_infos is None: - self.parent.mcrit_interface.queryAllSampleEntries() - return self.parent.family_infos is not None and self.parent.sample_infos is not None + return self.parent.mcrit_interface.ensureRemoteInformation() def _show_remote_cache_unavailable(self): self.clearTable() diff --git a/mcrit_plugin/ui_qt/widgets/MainWidget.py b/mcrit_plugin/ui_qt/widgets/MainWidget.py index 741bde6..3db5fe6 100644 --- a/mcrit_plugin/ui_qt/widgets/MainWidget.py +++ b/mcrit_plugin/ui_qt/widgets/MainWidget.py @@ -274,16 +274,18 @@ def _onConvertSmdaButtonClicked(self): self._buildLocalSmdaReport(self._applyConvertedReport, self._convertAndFetchRemote) def _convertAndFetchRemote(self): - local_smda_report = self.getLocalSmdaReport() - if self.parent.local_smda_report is None: - self.parent.getRemoteSampleInformation() - return local_smda_report + return self.getLocalSmdaReport() def _applyConvertedReport(self, local_smda_report): if self.parent.local_smda_report is None: self.parent.local_smda_report = local_smda_report - self.parent.local_widget.updateActivityInfo( - "Downloaded all family/sample information from MCRIT" + # the lists are needed once matches show up, so convert does not wait for them + self.cc.backend.run_request( + "MCRIT: downloading family and sample lists", + self.parent.getRemoteSampleInformation, + lambda _result: self.parent.local_widget.updateActivityInfo( + "Downloaded all family/sample information from MCRIT" + ), ) if self.parent.local_smda_report is not None: self.exportSmdaAction.setEnabled(True) @@ -301,16 +303,30 @@ def _applyConvertedReport(self, local_smda_report): ) # else query for family, version, library instead else: - dialog = self.SmdaInfoDialog(self) - dialog.exec_() - smda_info = dialog.getSmdaInfo() - self.parent.local_smda_report.family = smda_info["family"] - self.parent.local_smda_report.version = smda_info["version"] - self.parent.local_smda_report.is_library = smda_info["is_library"] - self.parent.block_match_widget.enable() - self.parent.function_match_widget.enable() + # the dialog offers the families, so they have to be here first + self.cc.backend.run_request( + "MCRIT: downloading family and sample lists", + self.parent.getRemoteSampleInformation, + lambda _result: self._askSmdaInfo(), + ) + return + self._enableMatchWidgets() self.parent.local_widget.update() + def _askSmdaInfo(self): + dialog = self.SmdaInfoDialog(self) + dialog.exec_() + smda_info = dialog.getSmdaInfo() + self.parent.local_smda_report.family = smda_info["family"] + self.parent.local_smda_report.version = smda_info["version"] + self.parent.local_smda_report.is_library = smda_info["is_library"] + self._enableMatchWidgets() + self.parent.local_widget.update() + + def _enableMatchWidgets(self): + self.parent.block_match_widget.enable() + self.parent.function_match_widget.enable() + def _onExportSmdaButtonClicked(self): self._buildLocalSmdaReport(self._exportReport) diff --git a/tests/core/test_mcrit_interface.py b/tests/core/test_mcrit_interface.py index 69177e6..311cc38 100644 --- a/tests/core/test_mcrit_interface.py +++ b/tests/core/test_mcrit_interface.py @@ -59,6 +59,7 @@ def _make_interface(timeout=10): inst.config = inst.parent.config inst._mcrit_server = "http://127.0.0.1:8000" inst._cache_lock = threading.Lock() + inst._remote_info_lock = threading.Lock() inst.mcrit_client = MagicMock() return inst @@ -168,3 +169,45 @@ def test_a_rejected_job_query_reports_failure(self): assert interface.queryJobs(sample_id=23) is None interface.parent.local_widget.updateActivityInfo.assert_called_with("Job query failed.") + + def test_jobs_of_a_sample_are_narrowed_by_the_server_and_checked_exactly(self): + interface = _make_interface() + + def job(parameters): + return SimpleNamespace(parameters=parameters) + + wanted = job("getMatchesForSample(23, 2)") + interface.mcrit_client.getQueueData.return_value = [ + wanted, + job("getMatchesForSample(230, 2)"), + job("updateMinHashesForSample(23)"), + ] + + assert interface.queryJobs(sample_id=23) == [wanted] + interface.mcrit_client.getQueueData.assert_called_once_with(filter="23") + + +class TestRemoteInformation: + def test_the_lists_are_downloaded_once_by_whoever_asks_first(self): + interface = _make_interface() + interface.parent.family_infos = None + interface.parent.sample_infos = None + interface.mcrit_client.getFamilies.return_value = {1: MagicMock()} + interface.mcrit_client.getSamples.return_value = {2: MagicMock()} + + assert interface.ensureRemoteInformation() is True + assert interface.ensureRemoteInformation() is True + assert interface.mcrit_client.getFamilies.call_count == 1 + assert interface.mcrit_client.getSamples.call_count == 1 + + def test_a_failed_download_is_tried_again_on_the_next_ask(self): + interface = _make_interface() + interface.parent.family_infos = None + interface.parent.sample_infos = None + interface.mcrit_client.getFamilies.return_value = None + interface.mcrit_client.getSamples.return_value = {2: MagicMock()} + + assert interface.ensureRemoteInformation() is False + interface.mcrit_client.getFamilies.return_value = {1: MagicMock()} + assert interface.ensureRemoteInformation() is True + assert interface.mcrit_client.getSamples.call_count == 1 From 55d1ef0478bc0dfd99b7f506817b884868e3265d Mon Sep 17 00:00:00 2001 From: r0ny123 <49360849+r0ny123@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:58:43 +0530 Subject: [PATCH 2/3] Look up the converted sample off the UI thread and ask for its family once After a conversion, _applyConvertedReport asked the server whether the sample is known, on the UI thread. Against mcrit.malpedia.io that froze Binary Ninja for 1.4 to 1.9 s on every Convert. The lookup now runs in the conversion's background work, _convertAndFetchRemote, which already existed for it. IDA runs that work synchronously behind its wait box as before. When the sample is unknown, the family dialog waits for the family list to download, and a second Convert click in that time queued a second dialog. Convert now stays blocked until the dialog closes. Convert on sample e0b6f853... in Binary Ninja 6.0 against mcrit.malpedia.io, until Fetch Matching Result is enabled: run before: duration / longest UI stall after 1 4.05 s / 1.88 s 3.46 s / 0.14 s 2 3.51 s / 1.43 s 3.52 s / 0.14 s 3 3.52 s / 1.45 s 3.41 s / 0.13 s --- CHANGELOG.md | 5 +++-- mcrit_plugin/ui_qt/widgets/MainWidget.py | 14 +++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0ba5c..db7966a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,9 @@ Ninja applies to that plugin only. downloading every job of the server and filtering them in the plugin. - Convert no longer waits for the family and sample lists: they download in the background after the report is built, side by side instead of one after the other, and one download serves every - widget that needs them. The family chooser for - an unknown sample opens once they are here. + widget that needs them. The family chooser for an unknown sample opens once they are here. On + Binary Ninja, Convert also asks whether the server knows the sample in the background, which + froze the UI for about 1.5 s against a remote server. - The bundled MCRIT client has the interface of MCRIT 1.12.0's client: `sample_group_only` only on the cross compare, the raising error modes, a default connect timeout, the job selectors of `getQueueData` and the unique-block parameters. The plugin sets its own timeout, so requests diff --git a/mcrit_plugin/ui_qt/widgets/MainWidget.py b/mcrit_plugin/ui_qt/widgets/MainWidget.py index 3db5fe6..6aa584a 100644 --- a/mcrit_plugin/ui_qt/widgets/MainWidget.py +++ b/mcrit_plugin/ui_qt/widgets/MainWidget.py @@ -274,7 +274,10 @@ def _onConvertSmdaButtonClicked(self): self._buildLocalSmdaReport(self._applyConvertedReport, self._convertAndFetchRemote) def _convertAndFetchRemote(self): - return self.getLocalSmdaReport() + local_smda_report = self.getLocalSmdaReport() + if local_smda_report is not None: + self.parent.mcrit_interface.querySampleSha256(local_smda_report.sha256) + return local_smda_report def _applyConvertedReport(self, local_smda_report): if self.parent.local_smda_report is None: @@ -291,9 +294,7 @@ def _applyConvertedReport(self, local_smda_report): self.exportSmdaAction.setEnabled(True) self.uploadSmdaAction.setEnabled(True) self.buildYaraStringAction.setEnabled(True) - # check if remote sample exists - self.parent.mcrit_interface.querySampleSha256(self.parent.local_smda_report.sha256) - # if yes, enable matching and use meta data + # if the conversion found the sample on the server, enable matching and use its meta data if self.parent.remote_sample_entry is not None: self.getMatchResultAction.setEnabled(True) self.parent.local_smda_report.family = self.parent.remote_sample_entry.family @@ -303,7 +304,9 @@ def _applyConvertedReport(self, local_smda_report): ) # else query for family, version, library instead else: - # the dialog offers the families, so they have to be here first + # the dialog offers the families, so they have to be here first; until it closes, + # another Convert would open a second one + self._building = True self.cc.backend.run_request( "MCRIT: downloading family and sample lists", self.parent.getRemoteSampleInformation, @@ -316,6 +319,7 @@ def _applyConvertedReport(self, local_smda_report): def _askSmdaInfo(self): dialog = self.SmdaInfoDialog(self) dialog.exec_() + self._building = False smda_info = dialog.getSmdaInfo() self.parent.local_smda_report.family = smda_info["family"] self.parent.local_smda_report.version = smda_info["version"] From 872623905b6268bc188e7c0d74c8d0127f209afd Mon Sep 17 00:00:00 2001 From: r0ny123 <49360849+r0ny123@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:58:19 +0530 Subject: [PATCH 3/3] Download the lists once on Convert, and unblock it after a failed request On the first Convert of a sample the server does not know, the family and sample lists were requested twice in a row: once in the background and once for the family chooser. The second waited for the first and, if the first had failed, downloaded both lists again, so against an unreachable server the chooser opened only after a second timeout. The background download now runs only when the chooser does not need the lists. _building, which keeps a second Convert from opening a second chooser, was cleared only by the chooser. A request that raises never reaches it, which left Convert, Export, Upload and the YARA builder answering that the report was still being built until a restart. The chooser's request now clears it when it raises. tests/ui_qt/test_convert_flow.py covers both branches and the failure. --- mcrit_plugin/ui_qt/widgets/MainWidget.py | 19 ++++- tests/ui_qt/test_convert_flow.py | 91 ++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 tests/ui_qt/test_convert_flow.py diff --git a/mcrit_plugin/ui_qt/widgets/MainWidget.py b/mcrit_plugin/ui_qt/widgets/MainWidget.py index 04c964e..aadef3f 100644 --- a/mcrit_plugin/ui_qt/widgets/MainWidget.py +++ b/mcrit_plugin/ui_qt/widgets/MainWidget.py @@ -280,9 +280,14 @@ def _convertAndFetchRemote(self): return local_smda_report def _applyConvertedReport(self, local_smda_report): - if self.parent.local_smda_report is None: + first_conversion = self.parent.local_smda_report is None + if first_conversion: self.parent.local_smda_report = local_smda_report - # the lists are needed once matches show up, so convert does not wait for them + if first_conversion and ( + self.parent.local_smda_report is None or self.parent.remote_sample_entry is not None + ): + # the lists are needed once matches show up, so convert does not wait for them; for + # an unknown sample the family chooser below downloads them instead self.cc.backend.run_request( "MCRIT: downloading family and sample lists", self.parent.getRemoteSampleInformation, @@ -309,13 +314,21 @@ def _applyConvertedReport(self, local_smda_report): self._building = True self.cc.backend.run_request( "MCRIT: downloading family and sample lists", - self.parent.getRemoteSampleInformation, + self._downloadListsForChooser, lambda _result: self._askSmdaInfo(), ) return self._enableMatchWidgets() self.parent.local_widget.update() + def _downloadListsForChooser(self): + try: + return self.parent.getRemoteSampleInformation() + except Exception: + # a raising request never reaches _askSmdaInfo; the next Convert may try again + self._building = False + raise + def _askSmdaInfo(self): dialog = self.SmdaInfoDialog(self) dialog.exec_() diff --git a/tests/ui_qt/test_convert_flow.py b/tests/ui_qt/test_convert_flow.py new file mode 100644 index 0000000..1369ea2 --- /dev/null +++ b/tests/ui_qt/test_convert_flow.py @@ -0,0 +1,91 @@ +"""After Convert, the family and sample lists download once, and the family chooser for an unknown +sample opens when they are here.""" + +import os +import types +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +pytest.importorskip("PySide6.QtWidgets") + +from mcrit_plugin.ui_qt.widgets.MainWidget import MainWidget # noqa: E402 + + +class _Backend: + def __init__(self): + self.pending = [] + + def run_request(self, title, work, on_done): + self.pending.append((work, on_done)) + + +def _main_widget(remote_sample_entry): + downloads = [] + widget = SimpleNamespace( + cc=SimpleNamespace(backend=_Backend()), + parent=SimpleNamespace( + local_smda_report=None, + remote_sample_entry=remote_sample_entry, + getRemoteSampleInformation=lambda: downloads.append(1) or True, + local_widget=MagicMock(), + ), + exportSmdaAction=MagicMock(), + uploadSmdaAction=MagicMock(), + buildYaraStringAction=MagicMock(), + getMatchResultAction=MagicMock(), + _building=False, + _enableMatchWidgets=MagicMock(), + _askSmdaInfo=MagicMock(), + ) + for name in ("_applyConvertedReport", "_downloadListsForChooser"): + setattr(widget, name, types.MethodType(getattr(MainWidget, name), widget)) + return widget, downloads + + +def _answer_all(backend): + while backend.pending: + work, on_done = backend.pending.pop(0) + on_done(work()) + + +def test_an_unknown_sample_downloads_the_lists_once_for_the_chooser(): + widget, downloads = _main_widget(remote_sample_entry=None) + + widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False)) + + assert len(widget.cc.backend.pending) == 1 + assert widget._building + _answer_all(widget.cc.backend) + assert downloads == [1] + widget._askSmdaInfo.assert_called_once() + + +def test_a_known_sample_downloads_the_lists_in_the_background(): + remote = SimpleNamespace(family="emotet", version="1", is_library=False) + widget, downloads = _main_widget(remote_sample_entry=remote) + + widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False)) + _answer_all(widget.cc.backend) + + assert downloads == [1] + assert not widget._building + widget._askSmdaInfo.assert_not_called() + assert widget.parent.local_smda_report.family == "emotet" + + +def test_a_failed_list_request_lets_the_next_convert_try_again(): + widget, _downloads = _main_widget(remote_sample_entry=None) + + def unreachable(): + raise RuntimeError("can't start new thread") + + widget.parent.getRemoteSampleInformation = unreachable + widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False)) + work, _on_done = widget.cc.backend.pending.pop() + + with pytest.raises(RuntimeError): + work() + assert not widget._building