Add KSA support to the indexing bot

.devcontainer/docker-compose.yml

@@ -19,11 +19,11 @@ services:
       SQS_TIMEOUT: 30
       STATUS_DB: DevMultiKANStatus
       XKAN_GHSECRET: test
-      INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
+      INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo ksa=InboundDevKsa.fifo
       MIRROR_SQS_QUEUE: MirroringDev.fifo
       STATUS_BUCKET: ckan-test-status
       STATUS_INTERVAL: 0
-      STATUS_KEYS: 'ksp=status/netkan.json ksp2=status/netkan-ksp2.json'
+      STATUS_KEYS: 'ksp=status/netkan.json ksp2=status/netkan-ksp2.json ksa=status/netkan-ksa.json'
       DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
       DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
     volumes:

dev-stack.py

@@ -28,6 +28,10 @@
                                     QueueName="InboundDevKsp2.fifo",
                                     ReceiveMessageWaitTimeSeconds=20,
                                     FifoQueue=True))
+inbound_ksa = t.add_resource(Queue("InboundDevKsa",
+                                   QueueName="InboundDevKsa.fifo",
+                                   ReceiveMessageWaitTimeSeconds=20,
+                                   FifoQueue=True))
 outbound = t.add_resource(Queue("OutboundDev",
                                 QueueName="OutboundDev.fifo",
                                 ReceiveMessageWaitTimeSeconds=20,
@@ -62,6 +66,7 @@
                 "Resource": [
                     GetAtt(inbound_ksp, "Arn"),
                     GetAtt(inbound_ksp2, "Arn"),
+                    GetAtt(inbound_ksa, "Arn"),
                     GetAtt(outbound, "Arn"),
                     GetAtt(addqueue, "Arn"),
                     GetAtt(mirrorqueue, "Arn"),
@@ -76,7 +81,7 @@
     }
 ))
 
-for queue in [inbound_ksp, inbound_ksp2, outbound, addqueue, mirrorqueue]:
+for queue in [inbound_ksp, inbound_ksp2, inbound_ksa, outbound, addqueue, mirrorqueue]:
     t.add_output([
         Output(
             "{}QueueURL".format(queue.title),

docker-compose.yml

@@ -35,7 +35,7 @@ services:
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
       AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
-      INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
+      INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo ksa=InboundDevKsa.fifo
       MAX_QUEUED: 1
       DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
       DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
@@ -72,6 +72,21 @@ services:
       # distros is 1000:1000, which will match the
       # user in the container.
       - ${HOME}/ckan_cache:/home/netkan/ckan_cache
+  inflator-ksa:
+    image: kspckan/inflator
+    environment:
+      QUEUES: InboundDevKsa.fifo,OutboundDev.fifo
+      GAME: KSA
+      GH_Token: ${CKAN_GH_Token}
+      AWS_REGION: ${CKAN_AWS_DEFAULT_REGION}
+      AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
+      AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
+    volumes:
+      # Docker will create this with root perms if
+      # it's not created first. First user on most
+      # distros is 1000:1000, which will match the
+      # user in the container.
+      - ${HOME}/ckan_cache:/home/netkan/ckan_cache
   webhooks:
     build:
       context: netkan/.
@@ -80,7 +95,7 @@ services:
       XKAN_GHSECRET: test
       SSH_KEY: ${CKAN_NETKAN_SSHKEY}
       NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
-      INFLATION_SQS_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
+      INFLATION_SQS_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo ksa=InboundDevKsa.fifo
       MIRROR_SQS_QUEUE: MirroringDev.fifo
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}

netkan/netkan/cli/common.py

@@ -75,6 +75,7 @@ class Game:
     MOD_ROOTS: Dict[str, str] = {
         'ksp':  'GameData',
         'ksp2': 'BepInEx/plugins',
+        'ksa':  'mods',
     }
 
     def __init__(self, name: str, shared: 'SharedArgs') -> None:

netkan/netkan/status.py

@@ -83,7 +83,7 @@ def export_game_mods(cls, condition: Condition, compat: bool = True) -> Dict[str
     @classmethod
     def export_all_mods(cls, compat: bool = True) -> Dict[str, Any]:
         data: Dict[str, Any] = {}
-        for game_id in ['ksp', 'ksp2']:
+        for game_id in ['ksp', 'ksp2', 'ksa']:
             data.update({game_id: cls.export_game_mods(
                 condition=ModStatus.game_id == game_id, compat=compat)})
         data.update({'no_game_id': cls.export_game_mods(

netkan/tests/cli.py

@@ -87,12 +87,24 @@ def test_shared_games_ksp2(self):
         shared.repos = ('ksp2=ckan',)
         self.assertEqual(shared.game_ids, ['ksp2'])
 
+    def test_shared_games_ksa(self):
+        shared = SharedArgs()
+        shared.ckanmeta_remotes = ('ksa=ckan_url',)
+        shared.repos = ('ksa=ckan',)
+        self.assertEqual(shared.game_ids, ['ksa'])
+
     def test_shared_games_multi(self):
         shared = SharedArgs()
         shared.ckanmeta_remotes = ('ksp2=ckan_url', 'ksp=ckan_url')
         shared.repos = ('ksp2=ckan', 'ksp2=ckan')
         self.assertEqual(shared.game_ids, ['ksp', 'ksp2'])
 
+    def test_shared_games_multi_with_ksa(self):
+        shared = SharedArgs()
+        shared.ckanmeta_remotes = ('ksa=ckan_url', 'ksp2=ckan_url', 'ksp=ckan_url')
+        shared.repos = ('ksa=ckan', 'ksp2=ckan')
+        self.assertEqual(shared.game_ids, ['ksa', 'ksp', 'ksp2'])
+
 
 class TestGame(SharedArgsHarness):
 
@@ -129,3 +141,21 @@ def test_netkan_remote_ksp2(self):
         path = f'{self.tmpdir.name}/upstream/netkan'
         self.assertEqual(
             Game('ksp2', self.shared_args).netkan_remote, path)
+
+    def test_shared_args_game_ksa(self):
+        self.assertEqual(self.shared_args.game('ksa').name, 'ksa')
+        self.assertIsInstance(self.shared_args.game('ksa'), Game)
+
+    def test_ckanmeta_remote_ksa(self):
+        path = f'{self.tmpdir.name}/upstream/ckan'
+        self.assertEqual(
+            Game('ksa', self.shared_args).ckanmeta_remote, path)
+
+    def test_netkan_remote_ksa(self):
+        path = f'{self.tmpdir.name}/upstream/netkan'
+        self.assertEqual(
+            Game('ksa', self.shared_args).netkan_remote, path)
+
+    def test_mod_root_ksa(self):
+        self.assertEqual(
+            Game('ksa', self.shared_args).mod_root, 'mods')

netkan/tests/common.py

@@ -76,15 +76,18 @@ def setUp(self):
         netkan_upstream = getattr(self, 'netkan_upstream')
         attributes = [
             ('ckanmeta_remotes',
-             (f'ksp={ckan_upstream}', f'ksp2={ckan_upstream}')),
+             (f'ksp={ckan_upstream}', f'ksp2={ckan_upstream}',
+              f'ksa={ckan_upstream}')),
             ('netkan_remotes',
-             (f'ksp={netkan_upstream}', f'ksp2={netkan_upstream}')),
+             (f'ksp={netkan_upstream}', f'ksp2={netkan_upstream}',
+              f'ksa={netkan_upstream}')),
             ('repos',
-             ('ksp=Test/KSP', 'ksp2=Test/KSP2')),
+             ('ksp=Test/KSP', 'ksp2=Test/KSP2', 'ksa=Test/KSA')),
             ('inflation_queues',
-             ('ksp=ksp.queue.url', 'ksp2=ksp2.queue.url')),
+             ('ksp=ksp.queue.url', 'ksp2=ksp2.queue.url',
+              'ksa=ksa.queue.url')),
             ('ia_collections',
-             ('ksp=kspcollection ksp2=ksp2collection'))
+             ('ksp=kspcollection', 'ksp2=ksp2collection', 'ksa=ksacollection'))
         ]
         for attr, val in attributes:
             setattr(self.shared_args, attr, val)

netkan/tests/indexer.py

@@ -337,3 +337,8 @@ def test_ksp2_message_append(self):
         indexer = IndexerQueueHandler(self.shared_args)
         indexer.append_message('ksp2', self.mocked_message())
         self.assertTrue('ksp2' in indexer.game_handlers)
+
+    def test_ksa_message_append(self):
+        indexer = IndexerQueueHandler(self.shared_args)
+        indexer.append_message('ksa', self.mocked_message())
+        self.assertTrue('ksa' in indexer.game_handlers)

netkan/tests/spacedock_adder.py

@@ -86,6 +86,11 @@ def test_ksp2_message_append(self):
         adder.append_message('ksp2', self.mocked_message())
         self.assertTrue('ksp2' in adder.game_handlers)
 
+    def test_ksa_message_append(self):
+        adder = SpaceDockAdderQueueHandler(self.shared_args)
+        adder.append_message('ksa', self.mocked_message())
+        self.assertTrue('ksa' in adder.game_handlers)
+
 
 class TestSpaceDockAdder(SharedArgsHarness):
 

netkan/tests/status.py

@@ -2,7 +2,7 @@
 # flake8: noqa
 
 from datetime import datetime
-from unittest import TestCase
+from unittest import TestCase, mock
 
 from netkan.status import ModStatus
 
@@ -49,3 +49,12 @@ def test_modidentifier(self):
         values = self.item_data()
         ModStatus.normalise_item('TheMod', values)
         self.assertEqual(values.get('ModIdentifier'), 'TheMod')
+
+
+class TestModStatusExport(TestCase):
+
+    @mock.patch.object(ModStatus, 'export_game_mods', return_value={})
+    def test_export_all_mods_games(self, mocked_export):
+        data = ModStatus.export_all_mods()
+        self.assertEqual(set(data.keys()),
+                         {'ksp', 'ksp2', 'ksa', 'no_game_id'})

netkan/tests/webhooks.py

@@ -36,8 +36,8 @@ def setUp(self) -> None:
             {
                 'LC_ALL': os.environ.get('LC_ALL', 'C.UTF-8'),
                 'LANG': os.environ.get('LANG', 'C.UTF-8'),
-                'NETKAN_REMOTES': f'ksp={netkan} ksp2={netkan}',
-                'CKANMETA_REMOTES': f'ksp={ckan} ksp2={ckan}',
+                'NETKAN_REMOTES': f'ksp={netkan} ksp2={netkan} ksa={netkan}',
+                'CKANMETA_REMOTES': f'ksp={ckan} ksp2={ckan} ksa={ckan}',
                 'SSH_KEY': '12345'
             },
             clear=True,
@@ -114,6 +114,21 @@ def test_inflate_ksp2(self, queued: MagicMock, sig: MagicMock):
         )
         self.assertEqual(call[2].get('QueueUrl'), 'ksp2.queue.url')
 
+    @mock.patch('netkan.webhooks.github_utils.sig_match')
+    @mock.patch('netkan.webhooks.config.WebhooksConfig.client')
+    def test_inflate_ksa(self, queued: MagicMock, sig: MagicMock):
+        sig.return_value = True
+        response = self.client.post(
+            '/gh/inflate/ksa', json=self.mock_netkan_hook(), follow_redirects=True)
+        self.assertEqual(response.status_code, 204)
+        call = queued.method_calls.pop().call_list().pop()
+        self.assertEqual(
+            call[2].get('Entries')[0].get('MessageAttributes').get(
+                'GameId').get('StringValue'),
+            'ksa',
+        )
+        self.assertEqual(call[2].get('QueueUrl'), 'ksa.queue.url')
+
     @mock.patch('netkan.webhooks.github_utils.sig_match')
     def test_inflate_ksp_wrong_branch(self, sig: MagicMock):
         sig.return_value = True
@@ -246,6 +261,20 @@ def test_inflate_ksp2(self, queued: MagicMock, sig: MagicMock):
             'ksp2',
         )
 
+    @mock.patch('netkan.webhooks.github_utils.sig_match')
+    @mock.patch('netkan.webhooks.config.WebhooksConfig.client')
+    def test_inflate_ksa(self, queued: MagicMock, sig: MagicMock):
+        sig.return_value = True
+        response = self.client.post(
+            '/gh/mirror/ksa', json=self.mock_netkan_hook(), follow_redirects=True)
+        self.assertEqual(response.status_code, 204)
+        call = queued.method_calls.pop().call_list().pop()
+        self.assertEqual(
+            call[2].get('Entries')[0].get('MessageAttributes').get(
+                'GameId').get('StringValue'),
+            'ksa',
+        )
+
     @mock.patch('netkan.webhooks.github_utils.sig_match')
     def test_inflate_ksp_wrong_branch(self, sig: MagicMock):
         sig.return_value = True
@@ -329,6 +358,19 @@ def test_inflate_ksp2(self, queued: MagicMock):
         )
         self.assertEqual(call[2].get('QueueUrl'), 'ksp2.queue.url')
 
+    @mock.patch('netkan.webhooks.config.WebhooksConfig.client')
+    def test_inflate_ksa(self, queued: MagicMock):
+        response = self.client.post(
+            '/inflate/ksa', json=self.mock_netkan_hook(), follow_redirects=True)
+        self.assertEqual(response.status_code, 204)
+        call = queued.method_calls.pop().call_list().pop()
+        self.assertEqual(
+            call[2].get('Entries')[0].get('MessageAttributes').get(
+                'GameId').get('StringValue'),
+            'ksa',
+        )
+        self.assertEqual(call[2].get('QueueUrl'), 'ksa.queue.url')
+
     def test_inflate_ksp_no_identifiers(self):
         data = self.mock_netkan_hook()
         data.update(identifiers=[])
@@ -387,6 +429,18 @@ def test_inflate_ksp2(self, queued: MagicMock):
             'ksp2',
         )
 
+    @mock.patch('netkan.webhooks.config.WebhooksConfig.client')
+    def test_inflate_ksa(self, queued: MagicMock):
+        response = self.client.post(
+            '/sd/inflate/ksa', data=self.mock_netkan_hook(), follow_redirects=True)
+        self.assertEqual(response.status_code, 204)
+        call = queued.method_calls.pop().call_list().pop()
+        self.assertEqual(
+            call[2].get('Entries')[0].get('MessageAttributes').get(
+                'GameId').get('StringValue'),
+            'ksa',
+        )
+
     def test_inflate_ksp_no_identifiers(self):
         data = self.mock_netkan_hook()
         data.update(mod_id='ABC')
@@ -471,3 +525,15 @@ def test_inflate_ksp2(self, queued: MagicMock):
                 'GameId').get('StringValue'),
             'ksp2',
         )
+
+    @mock.patch('netkan.webhooks.config.WebhooksConfig.client')
+    def test_inflate_ksa(self, queued: MagicMock):
+        response = self.client.post(
+            '/sd/add/ksa', data=self.mock_netkan_hook(), follow_redirects=True)
+        self.assertEqual(response.status_code, 204)
+        call = queued.method_calls.pop().call_list().pop()
+        self.assertEqual(
+            call[2].get('Entries')[0].get('MessageAttributes').get(
+                'GameId').get('StringValue'),
+            'ksa',
+        )