Skip to content
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,13 @@ Ninja applies to that plugin only.
matches of a result are grouped by function once, and each filter is aggregated once for the
current threshold, so with 221,000 matches over 705 functions clicking through the filters takes
about 0.15 s instead of 3 to 4 s.
- Choosing a matching job asks the server for the jobs that mention the sample, instead of
downloading every job of the server and filtering them in the plugin.
- Convert no longer waits for the family and sample lists: they download in the background after
the report is built, side by side instead of one after the other, and one download serves every
widget that needs them. The family chooser for an unknown sample opens once they are here. On
Binary Ninja, Convert also asks whether the server knows the sample in the background, which
froze the UI for about 1.5 s against a remote server.

## [2.0.0] - 2026-09-28

Expand Down
36 changes: 30 additions & 6 deletions mcrit_plugin/core/McritInterface.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ def __init__(self, parent, backend):
self.backend = backend
self.config = parent.config
self._cache_lock = threading.Lock()
self._remote_info_lock = threading.Lock()
self._mcrit_server = self.config.MCRIT_SERVER
self.mcrit_client = McritClient(self.config.MCRIT_SERVER)
# setTimeout maps 0 to no limit; skipping it would keep the client's 10 s connect timeout
Expand Down Expand Up @@ -189,8 +190,12 @@ def queryJobs(self, sample_id=None):
else:
self.parent.local_widget.updateActivityInfo("Querying jobs.")
try:
# fetch jobs
jobs = self.mcrit_client.getQueueData(filter="Matches")
# fetch jobs; the server narrows them by the sample id as text, which also matches
# other numbers, and older servers do not select by sample_ids, so the exact test
# below stays
jobs = self.mcrit_client.getQueueData(
filter="Matches" if sample_id is None else str(sample_id)
)
if jobs is None:
self.parent.local_widget.updateActivityInfo("Job query failed.")
return None
Expand All @@ -199,10 +204,13 @@ def queryJobs(self, sample_id=None):
jobs = [
job
for job in jobs
if "(" + str(sample_id) + ")" in job.parameters
or "(" + str(sample_id) + "," in job.parameters
or "," + str(sample_id) + "," in job.parameters
or "," + str(sample_id) + ")" in job.parameters
if "Matches" in job.parameters
and (
"(" + str(sample_id) + ")" in job.parameters
or "(" + str(sample_id) + "," in job.parameters
or "," + str(sample_id) + "," in job.parameters
or "," + str(sample_id) + ")" in job.parameters
)
]
if jobs:
self.parent.local_widget.updateActivityInfo("Success! Fetched Jobs.")
Expand Down Expand Up @@ -246,6 +254,22 @@ def getMatchingJobById(self, job_id):
except Exception as exc:
self._reportFailure("Result query", exc)

def ensureRemoteInformation(self):
"""Download the family and sample lists unless the session has them. Callable from several
threads: one downloads while the others wait for it instead of repeating it."""
with self._remote_info_lock:
# the two lists are independent, so they download side by side
downloads = []
if self.parent.family_infos is None:
downloads.append(threading.Thread(target=self.queryAllFamilyEntries, daemon=True))
if self.parent.sample_infos is None:
downloads.append(threading.Thread(target=self.queryAllSampleEntries, daemon=True))
for download in downloads:
download.start()
for download in downloads:
download.join()
return self.parent.family_infos is not None and self.parent.sample_infos is not None

def queryAllFamilyEntries(self):
self.parent.local_widget.updateActivityInfo("Querying for FamilyEntries")
try:
Expand Down
5 changes: 1 addition & 4 deletions mcrit_plugin/ui_qt/McritSession.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,7 @@ def getRemoteSampleInformation(self):
"""Download family/sample meta data; callable off the UI thread, so it touches no widget."""
time_before = time.time()
print("[/] starting download of meta data from MCRIT...")
self.mcrit_interface.queryAllFamilyEntries()
print("[|] downloaded FamilyEntries!")
self.mcrit_interface.queryAllSampleEntries()
print("[|] downloaded SampleEntries!")
self.mcrit_interface.ensureRemoteInformation()
print("[\\] this took %3.2f seconds.\n" % (time.time() - time_before))

def setupWidgets(self):
Expand Down
6 changes: 1 addition & 5 deletions mcrit_plugin/ui_qt/widgets/BlockMatchWidget.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,11 +147,7 @@ def _get_family_entry(self, family_id):

def _fetch_remote_cache(self):
"""Request the family and sample lists if they are missing; touches no widget."""
if self.parent.family_infos is None:
self.parent.mcrit_interface.queryAllFamilyEntries()
if self.parent.sample_infos is None:
self.parent.mcrit_interface.queryAllSampleEntries()
return self.parent.family_infos is not None and self.parent.sample_infos is not None
return self.parent.mcrit_interface.ensureRemoteInformation()

def _show_remote_cache_unavailable(self):
self.clearTable()
Expand Down
6 changes: 1 addition & 5 deletions mcrit_plugin/ui_qt/widgets/FunctionMatchWidget.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,7 @@ def _get_family_entry(self, family_id):

def _fetch_remote_cache(self):
"""Request the family and sample lists if they are missing; touches no widget."""
if self.parent.family_infos is None:
self.parent.mcrit_interface.queryAllFamilyEntries()
if self.parent.sample_infos is None:
self.parent.mcrit_interface.queryAllSampleEntries()
return self.parent.family_infos is not None and self.parent.sample_infos is not None
return self.parent.mcrit_interface.ensureRemoteInformation()

def _show_remote_cache_unavailable(self):
self.clearTable()
Expand Down
65 changes: 49 additions & 16 deletions mcrit_plugin/ui_qt/widgets/MainWidget.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,23 +275,31 @@ def _onConvertSmdaButtonClicked(self):

def _convertAndFetchRemote(self):
local_smda_report = self.getLocalSmdaReport()
if self.parent.local_smda_report is None:
self.parent.getRemoteSampleInformation()
if local_smda_report is not None:
self.parent.mcrit_interface.querySampleSha256(local_smda_report.sha256)
return local_smda_report

def _applyConvertedReport(self, local_smda_report):
if self.parent.local_smda_report is None:
first_conversion = self.parent.local_smda_report is None
if first_conversion:
self.parent.local_smda_report = local_smda_report
self.parent.local_widget.updateActivityInfo(
"Downloaded all family/sample information from MCRIT"
if first_conversion and (
self.parent.local_smda_report is None or self.parent.remote_sample_entry is not None
):
# the lists are needed once matches show up, so convert does not wait for them; for
# an unknown sample the family chooser below downloads them instead
self.cc.backend.run_request(
"MCRIT: downloading family and sample lists",
self.parent.getRemoteSampleInformation,
lambda _result: self.parent.local_widget.updateActivityInfo(
"Downloaded all family/sample information from MCRIT"
),
)
if self.parent.local_smda_report is not None:
self.exportSmdaAction.setEnabled(True)
self.uploadSmdaAction.setEnabled(True)
self.buildYaraStringAction.setEnabled(True)
# check if remote sample exists
self.parent.mcrit_interface.querySampleSha256(self.parent.local_smda_report.sha256)
# if yes, enable matching and use meta data
# if the conversion found the sample on the server, enable matching and use its meta data
if self.parent.remote_sample_entry is not None:
self.getMatchResultAction.setEnabled(True)
self.parent.local_smda_report.family = self.parent.remote_sample_entry.family
Expand All @@ -301,16 +309,41 @@ def _applyConvertedReport(self, local_smda_report):
)
# else query for family, version, library instead
else:
dialog = self.SmdaInfoDialog(self)
dialog.exec_()
smda_info = dialog.getSmdaInfo()
self.parent.local_smda_report.family = smda_info["family"]
self.parent.local_smda_report.version = smda_info["version"]
self.parent.local_smda_report.is_library = smda_info["is_library"]
self.parent.block_match_widget.enable()
self.parent.function_match_widget.enable()
# the dialog offers the families, so they have to be here first; until it closes,
# another Convert would open a second one
self._building = True
self.cc.backend.run_request(
"MCRIT: downloading family and sample lists",
self._downloadListsForChooser,
lambda _result: self._askSmdaInfo(),
)
return
self._enableMatchWidgets()
self.parent.local_widget.update()

def _downloadListsForChooser(self):
try:
return self.parent.getRemoteSampleInformation()
except Exception:
# a raising request never reaches _askSmdaInfo; the next Convert may try again
self._building = False
raise

def _askSmdaInfo(self):
dialog = self.SmdaInfoDialog(self)
dialog.exec_()
self._building = False
smda_info = dialog.getSmdaInfo()
self.parent.local_smda_report.family = smda_info["family"]
self.parent.local_smda_report.version = smda_info["version"]
self.parent.local_smda_report.is_library = smda_info["is_library"]
self._enableMatchWidgets()
self.parent.local_widget.update()

def _enableMatchWidgets(self):
self.parent.block_match_widget.enable()
self.parent.function_match_widget.enable()

def _onExportSmdaButtonClicked(self):
self._buildLocalSmdaReport(self._exportReport)

Expand Down
43 changes: 43 additions & 0 deletions tests/core/test_mcrit_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def _make_interface(timeout=10):
inst.config = inst.parent.config
inst._mcrit_server = "http://127.0.0.1:8000"
inst._cache_lock = threading.Lock()
inst._remote_info_lock = threading.Lock()
inst.mcrit_client = MagicMock()
return inst

Expand Down Expand Up @@ -195,6 +196,48 @@ def test_a_rejected_job_query_reports_failure(self):
assert interface.queryJobs(sample_id=23) is None
interface.parent.local_widget.updateActivityInfo.assert_called_with("Job query failed.")

def test_jobs_of_a_sample_are_narrowed_by_the_server_and_checked_exactly(self):
interface = _make_interface()

def job(parameters):
return SimpleNamespace(parameters=parameters)

wanted = job("getMatchesForSample(23, 2)")
interface.mcrit_client.getQueueData.return_value = [
wanted,
job("getMatchesForSample(230, 2)"),
job("updateMinHashesForSample(23)"),
]

assert interface.queryJobs(sample_id=23) == [wanted]
interface.mcrit_client.getQueueData.assert_called_once_with(filter="23")


class TestRemoteInformation:
def test_the_lists_are_downloaded_once_by_whoever_asks_first(self):
interface = _make_interface()
interface.parent.family_infos = None
interface.parent.sample_infos = None
interface.mcrit_client.getFamilies.return_value = {1: MagicMock()}
interface.mcrit_client.getSamples.return_value = {2: MagicMock()}

assert interface.ensureRemoteInformation() is True
assert interface.ensureRemoteInformation() is True
assert interface.mcrit_client.getFamilies.call_count == 1
assert interface.mcrit_client.getSamples.call_count == 1

def test_a_failed_download_is_tried_again_on_the_next_ask(self):
interface = _make_interface()
interface.parent.family_infos = None
interface.parent.sample_infos = None
interface.mcrit_client.getFamilies.return_value = None
interface.mcrit_client.getSamples.return_value = {2: MagicMock()}

assert interface.ensureRemoteInformation() is False
interface.mcrit_client.getFamilies.return_value = {1: MagicMock()}
assert interface.ensureRemoteInformation() is True
assert interface.mcrit_client.getSamples.call_count == 1


class TestQueryRemoteFunction:
def _interface(self, sample_infos=None):
Expand Down
91 changes: 91 additions & 0 deletions tests/ui_qt/test_convert_flow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
"""After Convert, the family and sample lists download once, and the family chooser for an unknown
sample opens when they are here."""

import os
import types
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest

os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
pytest.importorskip("PySide6.QtWidgets")

from mcrit_plugin.ui_qt.widgets.MainWidget import MainWidget # noqa: E402


class _Backend:
def __init__(self):
self.pending = []

def run_request(self, title, work, on_done):
self.pending.append((work, on_done))


def _main_widget(remote_sample_entry):
downloads = []
widget = SimpleNamespace(
cc=SimpleNamespace(backend=_Backend()),
parent=SimpleNamespace(
local_smda_report=None,
remote_sample_entry=remote_sample_entry,
getRemoteSampleInformation=lambda: downloads.append(1) or True,
local_widget=MagicMock(),
),
exportSmdaAction=MagicMock(),
uploadSmdaAction=MagicMock(),
buildYaraStringAction=MagicMock(),
getMatchResultAction=MagicMock(),
_building=False,
_enableMatchWidgets=MagicMock(),
_askSmdaInfo=MagicMock(),
)
for name in ("_applyConvertedReport", "_downloadListsForChooser"):
setattr(widget, name, types.MethodType(getattr(MainWidget, name), widget))
return widget, downloads


def _answer_all(backend):
while backend.pending:
work, on_done = backend.pending.pop(0)
on_done(work())


def test_an_unknown_sample_downloads_the_lists_once_for_the_chooser():
widget, downloads = _main_widget(remote_sample_entry=None)

widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False))

assert len(widget.cc.backend.pending) == 1
assert widget._building
_answer_all(widget.cc.backend)
assert downloads == [1]
widget._askSmdaInfo.assert_called_once()


def test_a_known_sample_downloads_the_lists_in_the_background():
remote = SimpleNamespace(family="emotet", version="1", is_library=False)
widget, downloads = _main_widget(remote_sample_entry=remote)

widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False))
_answer_all(widget.cc.backend)

assert downloads == [1]
assert not widget._building
widget._askSmdaInfo.assert_not_called()
assert widget.parent.local_smda_report.family == "emotet"


def test_a_failed_list_request_lets_the_next_convert_try_again():
widget, _downloads = _main_widget(remote_sample_entry=None)

def unreachable():
raise RuntimeError("can't start new thread")

widget.parent.getRemoteSampleInformation = unreachable
widget._applyConvertedReport(SimpleNamespace(family="", version="", is_library=False))
work, _on_done = widget.cc.backend.pending.pop()

with pytest.raises(RuntimeError):
work()
assert not widget._building
Loading