diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 5ba7136..3456262 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -19,11 +19,11 @@ services: SQS_TIMEOUT: 30 STATUS_DB: DevMultiKANStatus XKAN_GHSECRET: test - INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo + INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo ksa=InboundDevKsa.fifo MIRROR_SQS_QUEUE: MirroringDev.fifo STATUS_BUCKET: ckan-test-status STATUS_INTERVAL: 0 - STATUS_KEYS: 'ksp=status/netkan.json ksp2=status/netkan-ksp2.json' + STATUS_KEYS: 'ksp=status/netkan.json ksp2=status/netkan-ksp2.json ksa=status/netkan-ksa.json' DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID} DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN} volumes: diff --git a/dev-stack.py b/dev-stack.py index a3a0acd..93d75f7 100644 --- a/dev-stack.py +++ b/dev-stack.py @@ -28,6 +28,10 @@ QueueName="InboundDevKsp2.fifo", ReceiveMessageWaitTimeSeconds=20, FifoQueue=True)) +inbound_ksa = t.add_resource(Queue("InboundDevKsa", + QueueName="InboundDevKsa.fifo", + ReceiveMessageWaitTimeSeconds=20, + FifoQueue=True)) outbound = t.add_resource(Queue("OutboundDev", QueueName="OutboundDev.fifo", ReceiveMessageWaitTimeSeconds=20, @@ -62,6 +66,7 @@ "Resource": [ GetAtt(inbound_ksp, "Arn"), GetAtt(inbound_ksp2, "Arn"), + GetAtt(inbound_ksa, "Arn"), GetAtt(outbound, "Arn"), GetAtt(addqueue, "Arn"), GetAtt(mirrorqueue, "Arn"), @@ -76,7 +81,7 @@ } )) -for queue in [inbound_ksp, inbound_ksp2, outbound, addqueue, mirrorqueue]: +for queue in [inbound_ksp, inbound_ksp2, inbound_ksa, outbound, addqueue, mirrorqueue]: t.add_output([ Output( "{}QueueURL".format(queue.title), diff --git a/docker-compose.yml b/docker-compose.yml index 6104340..50096e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION} AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY} AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID} - INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo + INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo ksa=InboundDevKsa.fifo MAX_QUEUED: 1 DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID} DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN} @@ -72,6 +72,21 @@ services: # distros is 1000:1000, which will match the # user in the container. - ${HOME}/ckan_cache:/home/netkan/ckan_cache + inflator-ksa: + image: kspckan/inflator + environment: + QUEUES: InboundDevKsa.fifo,OutboundDev.fifo + GAME: KSA + GH_Token: ${CKAN_GH_Token} + AWS_REGION: ${CKAN_AWS_DEFAULT_REGION} + AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY} + AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID} + volumes: + # Docker will create this with root perms if + # it's not created first. First user on most + # distros is 1000:1000, which will match the + # user in the container. + - ${HOME}/ckan_cache:/home/netkan/ckan_cache webhooks: build: context: netkan/. @@ -80,7 +95,7 @@ services: XKAN_GHSECRET: test SSH_KEY: ${CKAN_NETKAN_SSHKEY} NETKAN_REMOTES: ${NETKAN_METADATA_PATHS} - INFLATION_SQS_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo + INFLATION_SQS_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo ksa=InboundDevKsa.fifo MIRROR_SQS_QUEUE: MirroringDev.fifo AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION} AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY} diff --git a/netkan/netkan/cli/common.py b/netkan/netkan/cli/common.py index 284460f..9e09691 100644 --- a/netkan/netkan/cli/common.py +++ b/netkan/netkan/cli/common.py @@ -75,6 +75,7 @@ class Game: MOD_ROOTS: Dict[str, str] = { 'ksp': 'GameData', 'ksp2': 'BepInEx/plugins', + 'ksa': 'mods', } def __init__(self, name: str, shared: 'SharedArgs') -> None: diff --git a/netkan/netkan/status.py b/netkan/netkan/status.py index 91404bf..17bee0c 100644 --- a/netkan/netkan/status.py +++ b/netkan/netkan/status.py @@ -83,7 +83,7 @@ def export_game_mods(cls, condition: Condition, compat: bool = True) -> Dict[str @classmethod def export_all_mods(cls, compat: bool = True) -> Dict[str, Any]: data: Dict[str, Any] = {} - for game_id in ['ksp', 'ksp2']: + for game_id in ['ksp', 'ksp2', 'ksa']: data.update({game_id: cls.export_game_mods( condition=ModStatus.game_id == game_id, compat=compat)}) data.update({'no_game_id': cls.export_game_mods( diff --git a/netkan/tests/cli.py b/netkan/tests/cli.py index 5e899e0..32c8a70 100644 --- a/netkan/tests/cli.py +++ b/netkan/tests/cli.py @@ -87,12 +87,24 @@ def test_shared_games_ksp2(self): shared.repos = ('ksp2=ckan',) self.assertEqual(shared.game_ids, ['ksp2']) + def test_shared_games_ksa(self): + shared = SharedArgs() + shared.ckanmeta_remotes = ('ksa=ckan_url',) + shared.repos = ('ksa=ckan',) + self.assertEqual(shared.game_ids, ['ksa']) + def test_shared_games_multi(self): shared = SharedArgs() shared.ckanmeta_remotes = ('ksp2=ckan_url', 'ksp=ckan_url') shared.repos = ('ksp2=ckan', 'ksp2=ckan') self.assertEqual(shared.game_ids, ['ksp', 'ksp2']) + def test_shared_games_multi_with_ksa(self): + shared = SharedArgs() + shared.ckanmeta_remotes = ('ksa=ckan_url', 'ksp2=ckan_url', 'ksp=ckan_url') + shared.repos = ('ksa=ckan', 'ksp2=ckan') + self.assertEqual(shared.game_ids, ['ksa', 'ksp', 'ksp2']) + class TestGame(SharedArgsHarness): @@ -129,3 +141,21 @@ def test_netkan_remote_ksp2(self): path = f'{self.tmpdir.name}/upstream/netkan' self.assertEqual( Game('ksp2', self.shared_args).netkan_remote, path) + + def test_shared_args_game_ksa(self): + self.assertEqual(self.shared_args.game('ksa').name, 'ksa') + self.assertIsInstance(self.shared_args.game('ksa'), Game) + + def test_ckanmeta_remote_ksa(self): + path = f'{self.tmpdir.name}/upstream/ckan' + self.assertEqual( + Game('ksa', self.shared_args).ckanmeta_remote, path) + + def test_netkan_remote_ksa(self): + path = f'{self.tmpdir.name}/upstream/netkan' + self.assertEqual( + Game('ksa', self.shared_args).netkan_remote, path) + + def test_mod_root_ksa(self): + self.assertEqual( + Game('ksa', self.shared_args).mod_root, 'mods') diff --git a/netkan/tests/common.py b/netkan/tests/common.py index 75b9ccf..0b4f643 100644 --- a/netkan/tests/common.py +++ b/netkan/tests/common.py @@ -76,15 +76,18 @@ def setUp(self): netkan_upstream = getattr(self, 'netkan_upstream') attributes = [ ('ckanmeta_remotes', - (f'ksp={ckan_upstream}', f'ksp2={ckan_upstream}')), + (f'ksp={ckan_upstream}', f'ksp2={ckan_upstream}', + f'ksa={ckan_upstream}')), ('netkan_remotes', - (f'ksp={netkan_upstream}', f'ksp2={netkan_upstream}')), + (f'ksp={netkan_upstream}', f'ksp2={netkan_upstream}', + f'ksa={netkan_upstream}')), ('repos', - ('ksp=Test/KSP', 'ksp2=Test/KSP2')), + ('ksp=Test/KSP', 'ksp2=Test/KSP2', 'ksa=Test/KSA')), ('inflation_queues', - ('ksp=ksp.queue.url', 'ksp2=ksp2.queue.url')), + ('ksp=ksp.queue.url', 'ksp2=ksp2.queue.url', + 'ksa=ksa.queue.url')), ('ia_collections', - ('ksp=kspcollection ksp2=ksp2collection')) + ('ksp=kspcollection', 'ksp2=ksp2collection', 'ksa=ksacollection')) ] for attr, val in attributes: setattr(self.shared_args, attr, val) diff --git a/netkan/tests/indexer.py b/netkan/tests/indexer.py index d11ed03..bb67173 100644 --- a/netkan/tests/indexer.py +++ b/netkan/tests/indexer.py @@ -337,3 +337,8 @@ def test_ksp2_message_append(self): indexer = IndexerQueueHandler(self.shared_args) indexer.append_message('ksp2', self.mocked_message()) self.assertTrue('ksp2' in indexer.game_handlers) + + def test_ksa_message_append(self): + indexer = IndexerQueueHandler(self.shared_args) + indexer.append_message('ksa', self.mocked_message()) + self.assertTrue('ksa' in indexer.game_handlers) diff --git a/netkan/tests/spacedock_adder.py b/netkan/tests/spacedock_adder.py index 82b6581..a4a07d6 100644 --- a/netkan/tests/spacedock_adder.py +++ b/netkan/tests/spacedock_adder.py @@ -86,6 +86,11 @@ def test_ksp2_message_append(self): adder.append_message('ksp2', self.mocked_message()) self.assertTrue('ksp2' in adder.game_handlers) + def test_ksa_message_append(self): + adder = SpaceDockAdderQueueHandler(self.shared_args) + adder.append_message('ksa', self.mocked_message()) + self.assertTrue('ksa' in adder.game_handlers) + class TestSpaceDockAdder(SharedArgsHarness): diff --git a/netkan/tests/status.py b/netkan/tests/status.py index 124419d..8283c18 100644 --- a/netkan/tests/status.py +++ b/netkan/tests/status.py @@ -2,7 +2,7 @@ # flake8: noqa from datetime import datetime -from unittest import TestCase +from unittest import TestCase, mock from netkan.status import ModStatus @@ -49,3 +49,12 @@ def test_modidentifier(self): values = self.item_data() ModStatus.normalise_item('TheMod', values) self.assertEqual(values.get('ModIdentifier'), 'TheMod') + + +class TestModStatusExport(TestCase): + + @mock.patch.object(ModStatus, 'export_game_mods', return_value={}) + def test_export_all_mods_games(self, mocked_export): + data = ModStatus.export_all_mods() + self.assertEqual(set(data.keys()), + {'ksp', 'ksp2', 'ksa', 'no_game_id'}) diff --git a/netkan/tests/webhooks.py b/netkan/tests/webhooks.py index f5607a3..540d1df 100644 --- a/netkan/tests/webhooks.py +++ b/netkan/tests/webhooks.py @@ -36,8 +36,8 @@ def setUp(self) -> None: { 'LC_ALL': os.environ.get('LC_ALL', 'C.UTF-8'), 'LANG': os.environ.get('LANG', 'C.UTF-8'), - 'NETKAN_REMOTES': f'ksp={netkan} ksp2={netkan}', - 'CKANMETA_REMOTES': f'ksp={ckan} ksp2={ckan}', + 'NETKAN_REMOTES': f'ksp={netkan} ksp2={netkan} ksa={netkan}', + 'CKANMETA_REMOTES': f'ksp={ckan} ksp2={ckan} ksa={ckan}', 'SSH_KEY': '12345' }, clear=True, @@ -114,6 +114,21 @@ def test_inflate_ksp2(self, queued: MagicMock, sig: MagicMock): ) self.assertEqual(call[2].get('QueueUrl'), 'ksp2.queue.url') + @mock.patch('netkan.webhooks.github_utils.sig_match') + @mock.patch('netkan.webhooks.config.WebhooksConfig.client') + def test_inflate_ksa(self, queued: MagicMock, sig: MagicMock): + sig.return_value = True + response = self.client.post( + '/gh/inflate/ksa', json=self.mock_netkan_hook(), follow_redirects=True) + self.assertEqual(response.status_code, 204) + call = queued.method_calls.pop().call_list().pop() + self.assertEqual( + call[2].get('Entries')[0].get('MessageAttributes').get( + 'GameId').get('StringValue'), + 'ksa', + ) + self.assertEqual(call[2].get('QueueUrl'), 'ksa.queue.url') + @mock.patch('netkan.webhooks.github_utils.sig_match') def test_inflate_ksp_wrong_branch(self, sig: MagicMock): sig.return_value = True @@ -246,6 +261,20 @@ def test_inflate_ksp2(self, queued: MagicMock, sig: MagicMock): 'ksp2', ) + @mock.patch('netkan.webhooks.github_utils.sig_match') + @mock.patch('netkan.webhooks.config.WebhooksConfig.client') + def test_inflate_ksa(self, queued: MagicMock, sig: MagicMock): + sig.return_value = True + response = self.client.post( + '/gh/mirror/ksa', json=self.mock_netkan_hook(), follow_redirects=True) + self.assertEqual(response.status_code, 204) + call = queued.method_calls.pop().call_list().pop() + self.assertEqual( + call[2].get('Entries')[0].get('MessageAttributes').get( + 'GameId').get('StringValue'), + 'ksa', + ) + @mock.patch('netkan.webhooks.github_utils.sig_match') def test_inflate_ksp_wrong_branch(self, sig: MagicMock): sig.return_value = True @@ -329,6 +358,19 @@ def test_inflate_ksp2(self, queued: MagicMock): ) self.assertEqual(call[2].get('QueueUrl'), 'ksp2.queue.url') + @mock.patch('netkan.webhooks.config.WebhooksConfig.client') + def test_inflate_ksa(self, queued: MagicMock): + response = self.client.post( + '/inflate/ksa', json=self.mock_netkan_hook(), follow_redirects=True) + self.assertEqual(response.status_code, 204) + call = queued.method_calls.pop().call_list().pop() + self.assertEqual( + call[2].get('Entries')[0].get('MessageAttributes').get( + 'GameId').get('StringValue'), + 'ksa', + ) + self.assertEqual(call[2].get('QueueUrl'), 'ksa.queue.url') + def test_inflate_ksp_no_identifiers(self): data = self.mock_netkan_hook() data.update(identifiers=[]) @@ -387,6 +429,18 @@ def test_inflate_ksp2(self, queued: MagicMock): 'ksp2', ) + @mock.patch('netkan.webhooks.config.WebhooksConfig.client') + def test_inflate_ksa(self, queued: MagicMock): + response = self.client.post( + '/sd/inflate/ksa', data=self.mock_netkan_hook(), follow_redirects=True) + self.assertEqual(response.status_code, 204) + call = queued.method_calls.pop().call_list().pop() + self.assertEqual( + call[2].get('Entries')[0].get('MessageAttributes').get( + 'GameId').get('StringValue'), + 'ksa', + ) + def test_inflate_ksp_no_identifiers(self): data = self.mock_netkan_hook() data.update(mod_id='ABC') @@ -471,3 +525,15 @@ def test_inflate_ksp2(self, queued: MagicMock): 'GameId').get('StringValue'), 'ksp2', ) + + @mock.patch('netkan.webhooks.config.WebhooksConfig.client') + def test_inflate_ksa(self, queued: MagicMock): + response = self.client.post( + '/sd/add/ksa', data=self.mock_netkan_hook(), follow_redirects=True) + self.assertEqual(response.status_code, 204) + call = queued.method_calls.pop().call_list().pop() + self.assertEqual( + call[2].get('Entries')[0].get('MessageAttributes').get( + 'GameId').get('StringValue'), + 'ksa', + ) diff --git a/prod-stack.py b/prod-stack.py index 75e766c..b836405 100644 --- a/prod-stack.py +++ b/prod-stack.py @@ -22,15 +22,19 @@ BOT_FQDN = 'netkan.ksp-ckan.space' EMAIL = 'domains@ksp-ckan.space' PARAM_NAMESPACE = '/NetKAN/Indexer/' -NETKAN_REMOTES = 'ksp=git@github.com:KSP-CKAN/NetKAN.git ksp2=git@github.com:KSP-CKAN/KSP2-NetKAN.git' +NETKAN_REMOTES = ('ksp=git@github.com:KSP-CKAN/NetKAN.git' + ' ksp2=git@github.com:KSP-CKAN/KSP2-NetKAN.git' + ' ksa=git@github.com:KSAModding/KSA-NetKAN.git') NETKAN_USER = 'KSP-CKAN' -NETKAN_REPOS = 'ksp=NetKAN ksp2=KSP2-NetKAN' -CKANMETA_REMOTES = 'ksp=git@github.com:KSP-CKAN/CKAN-meta.git ksp2=git@github.com:KSP-CKAN/KSP2-CKAN-meta.git' +NETKAN_REPOS = 'ksp=NetKAN ksp2=KSP2-NetKAN ksa=KSA-NetKAN' +CKANMETA_REMOTES = ('ksp=git@github.com:KSP-CKAN/CKAN-meta.git' + ' ksp2=git@github.com:KSP-CKAN/KSP2-CKAN-meta.git' + ' ksa=git@github.com:KSAModding/KSA-CKAN-meta.git') CKANMETA_USER = 'KSP-CKAN' -CKANMETA_REPOS = 'ksp=CKAN-meta ksp2=KSP2-CKAN-meta' +CKANMETA_REPOS = 'ksp=CKAN-meta ksp2=KSP2-CKAN-meta ksa=KSA-CKAN-meta' NETKAN_USER = 'KSP-CKAN' STATUS_BUCKET = 'status.ksp-ckan.space' -STATUS_KEYS = 'ksp=status/netkan.json ksp2=status/netkan-ksp2.json' +STATUS_KEYS = 'ksp=status/netkan.json ksp2=status/netkan-ksp2.json ksa=status/netkan-ksa.json' if not ZONE_ID: print('Zone ID Required from EnvVar `CKAN_ZONEID`') @@ -51,6 +55,10 @@ QueueName="InboundKsp2.fifo", ReceiveMessageWaitTimeSeconds=20, FifoQueue=True)) +inbound_ksa = t.add_resource(Queue("NetKANKSAInbound", + QueueName="InboundKsa.fifo", + ReceiveMessageWaitTimeSeconds=20, + FifoQueue=True)) outbound = t.add_resource(Queue("NetKANOutbound", QueueName="Outbound.fifo", ReceiveMessageWaitTimeSeconds=20, @@ -65,7 +73,7 @@ FifoQueue=True)) -for queue in [inbound, inbound2, outbound, addqueue, mirrorqueue]: +for queue in [inbound, inbound2, inbound_ksa, outbound, addqueue, mirrorqueue]: t.add_output([ Output( "{}QueueURL".format(queue.title), @@ -79,8 +87,9 @@ ), ]) -INFLATION_QUEUES = Sub('ksp=${ksp} ksp2=${ksp2}', ksp=GetAtt( - inbound, 'QueueName'), ksp2=GetAtt(inbound2, 'QueueName')) +INFLATION_QUEUES = Sub('ksp=${ksp} ksp2=${ksp2} ksa=${ksa}', ksp=GetAtt( + inbound, 'QueueName'), ksp2=GetAtt(inbound2, 'QueueName'), + ksa=GetAtt(inbound_ksa, 'QueueName')) # DyanamoDB: MultiKAN Status multikan_db = t.add_resource(Table( @@ -165,6 +174,7 @@ "Resource": [ GetAtt(inbound, "Arn"), GetAtt(inbound2, "Arn"), + GetAtt(inbound_ksa, "Arn"), GetAtt(outbound, "Arn"), GetAtt(addqueue, "Arn"), GetAtt(mirrorqueue, "Arn"), @@ -353,7 +363,7 @@ # redeployment of services. ksp_builder_group = t.add_resource(Group("KspCkanBuilderGroup")) builder_services = [] -for service in ['Indexer', 'InflatorKsp', 'InflatorKsp2', 'Webhooks', 'Adder', 'Mirrorer']: +for service in ['Indexer', 'InflatorKsp', 'InflatorKsp2', 'InflatorKsa', 'Webhooks', 'Adder', 'Mirrorer']: builder_services.append( Sub( 'arn:aws:ecs:${AWS::Region}:${AWS::AccountId}:service/NetKANCluster/${service}', @@ -444,6 +454,7 @@ "Resource": [ GetAtt(inbound, "Arn"), GetAtt(inbound2, "Arn"), + GetAtt(inbound_ksa, "Arn"), ] }, { @@ -723,6 +734,26 @@ ('ckan_cache', '/home/netkan/ckan_cache') ] }, + { + 'name': 'InflatorKsa', + 'image': 'kspckan/inflator', + 'memory': '256', + 'secrets': ['GH_Token'], + 'env': [ + ( + 'QUEUES', Sub( + '${Inbound},${Outbound}', + Inbound=GetAtt(inbound_ksa, 'QueueName'), + Outbound=GetAtt(outbound, 'QueueName') + ) + ), + ('AWS_REGION', Sub('${AWS::Region}')), + ('GAME', 'KSA') + ], + 'volumes': [ + ('ckan_cache', '/home/netkan/ckan_cache') + ] + }, { 'name': 'StatusDumper', 'command': 'export-status-s3',