diff --git a/CHANGELOG.md b/CHANGELOG.md index fe11ded..adda1aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,13 @@ Ninja applies to that plugin only. matches of a result are grouped by function once, and each filter is aggregated once for the current threshold, so with 221,000 matches over 705 functions clicking through the filters takes about 0.15 s instead of 3 to 4 s. +- Choosing a matching job asks the server for the jobs that mention the sample, instead of + downloading every job of the server and filtering them in the plugin. +- Convert no longer waits for the family and sample lists: they download in the background after + the report is built, side by side instead of one after the other, and one download serves every + widget that needs them. The family chooser for an unknown sample opens once they are here. On + Binary Ninja, Convert also asks whether the server knows the sample in the background, which + froze the UI for about 1.5 s against a remote server. ## [2.0.0] - 2026-09-28 diff --git a/mcrit_plugin/core/McritInterface.py b/mcrit_plugin/core/McritInterface.py index 0445d05..4edab9a 100644 --- a/mcrit_plugin/core/McritInterface.py +++ b/mcrit_plugin/core/McritInterface.py @@ -25,6 +25,7 @@ def __init__(self, parent, backend): self.backend = backend self.config = parent.config self._cache_lock = threading.Lock() + self._remote_info_lock = threading.Lock() self._mcrit_server = self.config.MCRIT_SERVER self.mcrit_client = McritClient(self.config.MCRIT_SERVER) # setTimeout maps 0 to no limit; skipping it would keep the client's 10 s connect timeout @@ -189,8 +190,12 @@ def queryJobs(self, sample_id=None): else: self.parent.local_widget.updateActivityInfo("Querying jobs.") try: - # fetch jobs - jobs = self.mcrit_client.getQueueData(filter="Matches") + # fetch jobs; the server narrows them by the sample id as text, which also matches + # other numbers, and older servers do not select by sample_ids, so the exact test + # below stays + jobs = self.mcrit_client.getQueueData( + filter="Matches" if sample_id is None else str(sample_id) + ) if jobs is None: self.parent.local_widget.updateActivityInfo("Job query failed.") return None @@ -199,10 +204,13 @@ def queryJobs(self, sample_id=None): jobs = [ job for job in jobs - if "(" + str(sample_id) + ")" in job.parameters - or "(" + str(sample_id) + "," in job.parameters - or "," + str(sample_id) + "," in job.parameters - or "," + str(sample_id) + ")" in job.parameters + if "Matches" in job.parameters + and ( + "(" + str(sample_id) + ")" in job.parameters + or "(" + str(sample_id) + "," in job.parameters + or "," + str(sample_id) + "," in job.parameters + or "," + str(sample_id) + ")" in job.parameters + ) ] if jobs: self.parent.local_widget.updateActivityInfo("Success! Fetched Jobs.") @@ -246,6 +254,22 @@ def getMatchingJobById(self, job_id): except Exception as exc: self._reportFailure("Result query", exc) + def ensureRemoteInformation(self): + """Download the family and sample lists unless the session has them. Callable from several + threads: one downloads while the others wait for it instead of repeating it.""" + with self._remote_info_lock: + # the two lists are independent, so they download side by side + downloads = [] + if self.parent.family_infos is None: + downloads.append(threading.Thread(target=self.queryAllFamilyEntries, daemon=True)) + if self.parent.sample_infos is None: + downloads.append(threading.Thread(target=self.queryAllSampleEntries, daemon=True)) + for download in downloads: + download.start() + for download in downloads: + download.join() + return self.parent.family_infos is not None and self.parent.sample_infos is not None + def queryAllFamilyEntries(self): self.parent.local_widget.updateActivityInfo("Querying for FamilyEntries") try: diff --git a/mcrit_plugin/ui_qt/McritSession.py b/mcrit_plugin/ui_qt/McritSession.py index bcd9291..542b59d 100644 --- a/mcrit_plugin/ui_qt/McritSession.py +++ b/mcrit_plugin/ui_qt/McritSession.py @@ -82,10 +82,7 @@ def getRemoteSampleInformation(self): """Download family/sample meta data; callable off the UI thread, so it touches no widget.""" time_before = time.time() print("[/] starting download of meta data from MCRIT...") - self.mcrit_interface.queryAllFamilyEntries() - print("[|] downloaded FamilyEntries!") - self.mcrit_interface.queryAllSampleEntries() - print("[|] downloaded SampleEntries!") + self.mcrit_interface.ensureRemoteInformation() print("[\\] this took %3.2f seconds.\n" % (time.time() - time_before)) def setupWidgets(self): diff --git a/mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py b/mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py index 03d8cb4..8701c94 100644 --- a/mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py +++ b/mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py @@ -147,11 +147,7 @@ def _get_family_entry(self, family_id): def _fetch_remote_cache(self): """Request the family and sample lists if they are missing; touches no widget.""" - if self.parent.family_infos is None: - self.parent.mcrit_interface.queryAllFamilyEntries() - if self.parent.sample_infos is None: - self.parent.mcrit_interface.queryAllSampleEntries() - return self.parent.family_infos is not None and self.parent.sample_infos is not None + return self.parent.mcrit_interface.ensureRemoteInformation() def _show_remote_cache_unavailable(self): self.clearTable() diff --git a/mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py b/mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py index 8b85206..e5e1aae 100644 --- a/mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py +++ b/mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py @@ -137,11 +137,7 @@ def _get_family_entry(self, family_id): def _fetch_remote_cache(self): """Request the family and sample lists if they are missing; touches no widget.""" - if self.parent.family_infos is None: - self.parent.mcrit_interface.queryAllFamilyEntries() - if self.parent.sample_infos is None: - self.parent.mcrit_interface.queryAllSampleEntries() - return self.parent.family_infos is not None and self.parent.sample_infos is not None + return self.parent.mcrit_interface.ensureRemoteInformation() def _show_remote_cache_unavailable(self): self.clearTable() diff --git a/mcrit_plugin/ui_qt/widgets/MainWidget.py b/mcrit_plugin/ui_qt/widgets/MainWidget.py index e305ca0..aadef3f 100644 --- a/mcrit_plugin/ui_qt/widgets/MainWidget.py +++ b/mcrit_plugin/ui_qt/widgets/MainWidget.py @@ -275,23 +275,31 @@ def _onConvertSmdaButtonClicked(self): def _convertAndFetchRemote(self): local_smda_report = self.getLocalSmdaReport() - if self.parent.local_smda_report is None: - self.parent.getRemoteSampleInformation() + if local_smda_report is not None: + self.parent.mcrit_interface.querySampleSha256(local_smda_report.sha256) return local_smda_report def _applyConvertedReport(self, local_smda_report): - if self.parent.local_smda_report is None: + first_conversion = self.parent.local_smda_report is None + if first_conversion: self.parent.local_smda_report = local_smda_report - self.parent.local_widget.updateActivityInfo( - "Downloaded all family/sample information from MCRIT" + if first_conversion and ( + self.parent.local_smda_report is None or self.parent.remote_sample_entry is not None + ): + # the lists are needed once matches show up, so convert does not wait for them; for + # an unknown sample the family chooser below downloads them instead + self.cc.backend.run_request( + "MCRIT: downloading family and sample lists", + self.parent.getRemoteSampleInformation, + lambda _result: self.parent.local_widget.updateActivityInfo( + "Downloaded all family/sample information from MCRIT" + ), ) if self.parent.local_smda_report is not None: self.exportSmdaAction.setEnabled(True) self.uploadSmdaAction.setEnabled(True) self.buildYaraStringAction.setEnabled(True) - # check if remote sample exists - self.parent.mcrit_interface.querySampleSha256(self.parent.local_smda_report.sha256) - # if yes, enable matching and use meta data + # if the conversion found the sample on the server, enable matching and use its meta data if self.parent.remote_sample_entry is not None: self.getMatchResultAction.setEnabled(True) self.parent.local_smda_report.family = self.parent.remote_sample_entry.family @@ -301,16 +309,41 @@ def _applyConvertedReport(self, local_smda_report): ) # else query for family, version, library instead else: - dialog = self.SmdaInfoDialog(self) - dialog.exec_() - smda_info = dialog.getSmdaInfo() - self.parent.local_smda_report.family = smda_info["family"] - self.parent.local_smda_report.version = smda_info["version"] - self.parent.local_smda_report.is_library = smda_info["is_library"] - self.parent.block_match_widget.enable() - self.parent.function_match_widget.enable() + # the dialog offers the families, so they have to be here first; until it closes, + # another Convert would open a second one + self._building = True + self.cc.backend.run_request( + "MCRIT: downloading family and sample lists", + self._downloadListsForChooser, + lambda _result: self._askSmdaInfo(), + ) + return + self._enableMatchWidgets() + self.parent.local_widget.update() + + def _downloadListsForChooser(self): + try: + return self.parent.getRemoteSampleInformation() + except Exception: + # a raising request never reaches _askSmdaInfo; the next Convert may try again + self._building = False + raise + + def _askSmdaInfo(self): + dialog = self.SmdaInfoDialog(self) + dialog.exec_() + self._building = False + smda_info = dialog.getSmdaInfo() + self.parent.local_smda_report.family = smda_info["family"] + self.parent.local_smda_report.version = smda_info["version"] + self.parent.local_smda_report.is_library = smda_info["is_library"] + self._enableMatchWidgets() self.parent.local_widget.update() + def _enableMatchWidgets(self): + self.parent.block_match_widget.enable() + self.parent.function_match_widget.enable() + def _onExportSmdaButtonClicked(self): self._buildLocalSmdaReport(self._exportReport) diff --git a/tests/core/test_mcrit_interface.py b/tests/core/test_mcrit_interface.py index dffe783..5455469 100644 --- a/tests/core/test_mcrit_interface.py +++ b/tests/core/test_mcrit_interface.py @@ -59,6 +59,7 @@ def _make_interface(timeout=10): inst.config = inst.parent.config inst._mcrit_server = "http://127.0.0.1:8000" inst._cache_lock = threading.Lock() + inst._remote_info_lock = threading.Lock() inst.mcrit_client = MagicMock() return inst @@ -195,6 +196,48 @@ def test_a_rejected_job_query_reports_failure(self): assert interface.queryJobs(sample_id=23) is None interface.parent.local_widget.updateActivityInfo.assert_called_with("Job query failed.") + def test_jobs_of_a_sample_are_narrowed_by_the_server_and_checked_exactly(self): + interface = _make_interface() + + def job(parameters): + return SimpleNamespace(parameters=parameters) + + wanted = job("getMatchesForSample(23, 2)") + interface.mcrit_client.getQueueData.return_value = [ + wanted, + job("getMatchesForSample(230, 2)"), + job("updateMinHashesForSample(23)"), + ] + + assert interface.queryJobs(sample_id=23) == [wanted] + interface.mcrit_client.getQueueData.assert_called_once_with(filter="23") + + +class TestRemoteInformation: + def test_the_lists_are_downloaded_once_by_whoever_asks_first(self): + interface = _make_interface() + interface.parent.family_infos = None + interface.parent.sample_infos = None + interface.mcrit_client.getFamilies.return_value = {1: MagicMock()} + interface.mcrit_client.getSamples.return_value = {2: MagicMock()} + + assert interface.ensureRemoteInformation() is True + assert interface.ensureRemoteInformation() is True + assert interface.mcrit_client.getFamilies.call_count == 1 + assert interface.mcrit_client.getSamples.call_count == 1 + + def test_a_failed_download_is_tried_again_on_the_next_ask(self): + interface = _make_interface() + interface.parent.family_infos = None + interface.parent.sample_infos = None + interface.mcrit_client.getFamilies.return_value = None + interface.mcrit_client.getSamples.return_value = {2: MagicMock()} + + assert interface.ensureRemoteInformation() is False + interface.mcrit_client.getFamilies.return_value = {1: MagicMock()} + assert interface.ensureRemoteInformation() is True + assert interface.mcrit_client.getSamples.call_count == 1 + class TestQueryRemoteFunction: def _interface(self, sample_infos=None): diff --git a/tests/ui_qt/test_convert_flow.py b/tests/ui_qt/test_convert_flow.py new file mode 100644 index 0000000..1369ea2 --- /dev/null +++ b/tests/ui_qt/test_convert_flow.py @@ -0,0 +1,91 @@ +"""After Convert, the family and sample lists download once, and the family chooser for an unknown +sample opens when they are here.""" + +import os +import types +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +pytest.importorskip("PySide6.QtWidgets") + +from mcrit_plugin.ui_qt.widgets.MainWidget import MainWidget # noqa: E402 + + +class _Backend: + def __init__(self): + self.pending = [] + + def run_request(self, title, work, on_done): + self.pending.append((work, on_done)) + + +def _main_widget(remote_sample_entry): + downloads = [] + widget = SimpleNamespace( + cc=SimpleNamespace(backend=_Backend()), + parent=SimpleNamespace( + local_smda_report=None, + remote_sample_entry=remote_sample_entry, + getRemoteSampleInformation=lambda: downloads.append(1) or True, + local_widget=MagicMock(), + ), + exportSmdaAction=MagicMock(), + uploadSmdaAction=MagicMock(), + buildYaraStringAction=MagicMock(), + getMatchResultAction=MagicMock(), + _building=False, + _enableMatchWidgets=MagicMock(), + _askSmdaInfo=MagicMock(), + ) + for name in ("_applyConvertedReport", "_downloadListsForChooser"): + setattr(widget, name, types.MethodType(getattr(MainWidget, name), widget)) + return widget, downloads + + +def _answer_all(backend): + while backend.pending: + work, on_done = backend.pending.pop(0) + on_done(work()) + + +def test_an_unknown_sample_downloads_the_lists_once_for_the_chooser(): + widget, downloads = _main_widget(remote_sample_entry=None) + + widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False)) + + assert len(widget.cc.backend.pending) == 1 + assert widget._building + _answer_all(widget.cc.backend) + assert downloads == [1] + widget._askSmdaInfo.assert_called_once() + + +def test_a_known_sample_downloads_the_lists_in_the_background(): + remote = SimpleNamespace(family="emotet", version="1", is_library=False) + widget, downloads = _main_widget(remote_sample_entry=remote) + + widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False)) + _answer_all(widget.cc.backend) + + assert downloads == [1] + assert not widget._building + widget._askSmdaInfo.assert_not_called() + assert widget.parent.local_smda_report.family == "emotet" + + +def test_a_failed_list_request_lets_the_next_convert_try_again(): + widget, _downloads = _main_widget(remote_sample_entry=None) + + def unreachable(): + raise RuntimeError("can't start new thread") + + widget.parent.getRemoteSampleInformation = unreachable + widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False)) + work, _on_done = widget.cc.backend.pending.pop() + + with pytest.raises(RuntimeError): + work() + assert not widget._building