From f1ffe5130704acb1f16c6e3f8005a8879b06e192 Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Wed, 11 Jun 2025 18:37:40 +0200 Subject: [PATCH 01/10] KAB-1127 feat: add flow tracking - create and update tools --- src/keboola_mcp_server/tools/flow.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/keboola_mcp_server/tools/flow.py b/src/keboola_mcp_server/tools/flow.py index eb02bd29..ac0fe786 100644 --- a/src/keboola_mcp_server/tools/flow.py +++ b/src/keboola_mcp_server/tools/flow.py @@ -9,7 +9,7 @@ from fastmcp import Context, FastMCP from pydantic import AliasChoices, BaseModel, Field -from keboola_mcp_server.client import JsonDict, KeboolaClient +from keboola_mcp_server.client import ORCHESTRATOR_COMPONENT_ID, JsonDict, KeboolaClient from keboola_mcp_server.errors import tool_errors from keboola_mcp_server.links import Link, ProjectLinksManager from keboola_mcp_server.mcp import with_session_state @@ -20,6 +20,7 @@ FlowTask, ReducedFlow, ) +from keboola_mcp_server.tools.components.tools import _set_cfg_creation_metadata, _set_cfg_update_metadata from keboola_mcp_server.tools.validation import validate_flow_configuration_against_schema LOG = logging.getLogger(__name__) @@ -127,6 +128,13 @@ async def create_flow( name=name, description=description, flow_configuration=flow_configuration # Direct configuration ) + new_flow = FlowConfigurationResponse.from_raw_config(new_raw_configuration) + await _set_cfg_creation_metadata( + client, + component_id=ORCHESTRATOR_COMPONENT_ID, + configuration_id=new_flow.configuration_id, + ) + flow_id = str(new_raw_configuration['id']) flow_name = new_raw_configuration['name'] flow_links = links_manager.get_flow_links(flow_id=flow_id, flow_name=flow_name) @@ -190,6 +198,13 @@ async def update_flow( change_description=change_description, flow_configuration=flow_configuration, # Direct configuration ) + updated_flow = FlowConfigurationResponse.from_raw_config(updated_raw_configuration) + await _set_cfg_update_metadata( + client, + component_id=ORCHESTRATOR_COMPONENT_ID, + configuration_id=updated_flow.configuration_id, + configuration_version=updated_flow.version, + ) flow_id = str(updated_raw_configuration['id']) flow_name = updated_raw_configuration['name'] From d30ad73f5507f2a6ce8447d595882bdb38e654bb Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Wed, 11 Jun 2025 18:37:54 +0200 Subject: [PATCH 02/10] KAB-1127 chore: update vers --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4cd21b7b..4cdab13d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "keboola-mcp-server" -version = "1.0.0" +version = "1.0.1" description = "MCP server for interacting with Keboola Connection" readme = "README.md" requires-python = ">=3.10" From 3b63880dcc708dcd361ef4c7f317d377d528bb55 Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Wed, 11 Jun 2025 18:40:26 +0200 Subject: [PATCH 03/10] KAB-1127 chore: enable production release --- .github/workflows/release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 994edd2d..f439735f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,10 +60,10 @@ jobs: image-tag: canary-orion-${{ github.sha }} github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} -# - name: Trigger image tag update for production -# uses: ./.github/actions/trigger-image-tag-update -# if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev.') -# with: -# helm-chart: "mcp-server" -# image-tag: production-${{ github.sha }} -# github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} + - name: Trigger image tag update for production + uses: ./.github/actions/trigger-image-tag-update + if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev.') + with: + helm-chart: "mcp-server" + image-tag: production-${{ github.sha }} + github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} From 969ec7d012edc37b63a0df748b7f7d224d49aa99 Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Wed, 11 Jun 2025 19:06:16 +0200 Subject: [PATCH 04/10] KAB-1127 test: add integtests checking existence of flow metadata after update and create --- integtests/test_flow_tools.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/integtests/test_flow_tools.py b/integtests/test_flow_tools.py index c7168487..65522786 100644 --- a/integtests/test_flow_tools.py +++ b/integtests/test_flow_tools.py @@ -2,7 +2,8 @@ from fastmcp import Context from integtests.conftest import ConfigDef -from keboola_mcp_server.client import KeboolaClient +from keboola_mcp_server.client import ORCHESTRATOR_COMPONENT_ID, KeboolaClient +from keboola_mcp_server.config import MetadataField from keboola_mcp_server.tools.flow import ( FlowToolResponse, create_flow, @@ -55,6 +56,7 @@ async def test_create_and_retrieve_flow(mcp_context: Context, configs: list[Conf tasks=tasks, ) flow_id = created.flow_id + client = KeboolaClient.from_state(mcp_context.session.state) try: assert isinstance(created, FlowToolResponse) assert created.description == flow_description @@ -68,6 +70,19 @@ async def test_create_and_retrieve_flow(mcp_context: Context, configs: list[Conf assert detail.phases[0].name == 'Extract' assert detail.phases[1].name == 'Transform' assert detail.tasks[0].task['componentId'] == configs[0].component_id + + # Verify the metadata - check that KBC.MCP.createdBy is set to 'true' + metadata = await client.storage_client.configuration_metadata_get( + component_id=ORCHESTRATOR_COMPONENT_ID, + configuration_id=flow_id + ) + + # Convert metadata list to dictionary for easier checking + # metadata is a list of dicts with 'key' and 'value' keys + assert isinstance(metadata, list) + metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} + assert MetadataField.CREATED_BY_MCP in metadata_dict + assert metadata_dict[MetadataField.CREATED_BY_MCP] == 'true' finally: client = KeboolaClient.from_state(mcp_context.session.state) await client.storage_client.flow_delete(flow_id, skip_trash=True) @@ -106,6 +121,7 @@ async def test_update_flow(mcp_context: Context, configs: list[ConfigDef]) -> No tasks=tasks, ) flow_id = created.flow_id + client = KeboolaClient.from_state(mcp_context.session.state) try: new_name = 'Updated Flow Name' new_description = 'Updated description.' @@ -123,6 +139,20 @@ async def test_update_flow(mcp_context: Context, configs: list[ConfigDef]) -> No assert updated.description == new_description assert updated.success is True assert len(updated.links) == 3 + + # Verify the metadata - check that KBC.MCP.updatedBy.version.{version} is set to 'true' + metadata = await client.storage_client.configuration_metadata_get( + component_id=ORCHESTRATOR_COMPONENT_ID, + configuration_id=flow_id + ) + + assert isinstance(metadata, list) + metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} + sync_flow = await get_flow_detail(mcp_context, configuration_id=flow_id) + updated_by_md_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{sync_flow.version}' + assert updated_by_md_key in metadata_dict + assert metadata_dict[updated_by_md_key] == 'true' + finally: client = KeboolaClient.from_state(mcp_context.session.state) await client.storage_client.flow_delete(flow_id, skip_trash=True) From 75888f8d72a42c9cf825ffefea9e3a2794cf601b Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Wed, 11 Jun 2025 19:12:27 +0200 Subject: [PATCH 05/10] KAB-1127 fix: fix release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f439735f..30373864 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -66,4 +66,4 @@ jobs: with: helm-chart: "mcp-server" image-tag: production-${{ github.sha }} - github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} + github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} \ No newline at end of file From f23d274616a8c95d94a6ce6a3008e3cbaea59c4f Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Wed, 11 Jun 2025 19:24:24 +0200 Subject: [PATCH 06/10] KAB-1127 fix: update update flow test --- integtests/test_flow_tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integtests/test_flow_tools.py b/integtests/test_flow_tools.py index 65522786..1d6ef192 100644 --- a/integtests/test_flow_tools.py +++ b/integtests/test_flow_tools.py @@ -148,8 +148,8 @@ async def test_update_flow(mcp_context: Context, configs: list[ConfigDef]) -> No assert isinstance(metadata, list) metadata_dict = {item['key']: item['value'] for item in metadata if isinstance(item, dict)} - sync_flow = await get_flow_detail(mcp_context, configuration_id=flow_id) - updated_by_md_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{sync_flow.version}' + sync_flow = await client.storage_client.flow_detail(flow_id) + updated_by_md_key = f'{MetadataField.UPDATED_BY_MCP_PREFIX}{sync_flow["version"]}' assert updated_by_md_key in metadata_dict assert metadata_dict[updated_by_md_key] == 'true' From 338023d6527153491ca8092323b7d9eaad9b2eec Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Thu, 12 Jun 2025 09:33:07 +0200 Subject: [PATCH 07/10] KAB-1127 chore: disable production release - we can infer from included components --- .github/workflows/release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 30373864..91dc7630 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,10 +60,10 @@ jobs: image-tag: canary-orion-${{ github.sha }} github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} - - name: Trigger image tag update for production - uses: ./.github/actions/trigger-image-tag-update - if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev.') - with: - helm-chart: "mcp-server" - image-tag: production-${{ github.sha }} - github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} \ No newline at end of file + # - name: Trigger image tag update for production + # uses: ./.github/actions/trigger-image-tag-update + # if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev.') + # with: + # helm-chart: "mcp-server" + # image-tag: production-${{ github.sha }} + # github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} \ No newline at end of file From 315541c2faaaefc152b2f2045b91839b2f27c973 Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Thu, 12 Jun 2025 10:00:39 +0200 Subject: [PATCH 08/10] KAB-1127 refactor: update code wrt reviews --- src/keboola_mcp_server/tools/flow.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/keboola_mcp_server/tools/flow.py b/src/keboola_mcp_server/tools/flow.py index ac0fe786..e6079733 100644 --- a/src/keboola_mcp_server/tools/flow.py +++ b/src/keboola_mcp_server/tools/flow.py @@ -128,15 +128,14 @@ async def create_flow( name=name, description=description, flow_configuration=flow_configuration # Direct configuration ) - new_flow = FlowConfigurationResponse.from_raw_config(new_raw_configuration) await _set_cfg_creation_metadata( client, component_id=ORCHESTRATOR_COMPONENT_ID, - configuration_id=new_flow.configuration_id, + configuration_id=str(new_raw_configuration['id']), ) flow_id = str(new_raw_configuration['id']) - flow_name = new_raw_configuration['name'] + flow_name = str(new_raw_configuration['name']) flow_links = links_manager.get_flow_links(flow_id=flow_id, flow_name=flow_name) tool_response = FlowToolResponse.model_validate(new_raw_configuration | {'links': flow_links}) @@ -198,16 +197,16 @@ async def update_flow( change_description=change_description, flow_configuration=flow_configuration, # Direct configuration ) - updated_flow = FlowConfigurationResponse.from_raw_config(updated_raw_configuration) + await _set_cfg_update_metadata( client, component_id=ORCHESTRATOR_COMPONENT_ID, - configuration_id=updated_flow.configuration_id, - configuration_version=updated_flow.version, + configuration_id=str(updated_raw_configuration['id']), + configuration_version=cast(int, updated_raw_configuration['version']), ) flow_id = str(updated_raw_configuration['id']) - flow_name = updated_raw_configuration['name'] + flow_name = str(updated_raw_configuration['name']) flow_links = links_manager.get_flow_links(flow_id=flow_id, flow_name=flow_name) tool_response = FlowToolResponse.model_validate(updated_raw_configuration | {'links': flow_links}) From a791f5162916243a5ba7ee66febe90305408eeef Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Thu, 12 Jun 2025 11:39:04 +0200 Subject: [PATCH 09/10] KAB-1127 revert: using disable release changes --- .github/workflows/release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 91dc7630..cb148a90 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,10 +60,10 @@ jobs: image-tag: canary-orion-${{ github.sha }} github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} - # - name: Trigger image tag update for production - # uses: ./.github/actions/trigger-image-tag-update - # if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev.') - # with: - # helm-chart: "mcp-server" - # image-tag: production-${{ github.sha }} - # github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} \ No newline at end of file +# - name: Trigger image tag update for production +# uses: ./.github/actions/trigger-image-tag-update +# if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref, '-dev.') +# with: +# helm-chart: "mcp-server" +# image-tag: production-${{ github.sha }} +# github-app-private-key: ${{ secrets.GITOPS_KBC_STACKS_TRIGGER_APP_PVK }} \ No newline at end of file From 7c51211159354362b2a50b973b0dc4707580103a Mon Sep 17 00:00:00 2001 From: mariankrotil Date: Thu, 12 Jun 2025 12:58:45 +0200 Subject: [PATCH 10/10] KAB-1127 test: up0date tests wrt reviews --- integtests/test_flow_tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integtests/test_flow_tools.py b/integtests/test_flow_tools.py index 1d6ef192..e77f5b61 100644 --- a/integtests/test_flow_tools.py +++ b/integtests/test_flow_tools.py @@ -84,7 +84,6 @@ async def test_create_and_retrieve_flow(mcp_context: Context, configs: list[Conf assert MetadataField.CREATED_BY_MCP in metadata_dict assert metadata_dict[MetadataField.CREATED_BY_MCP] == 'true' finally: - client = KeboolaClient.from_state(mcp_context.session.state) await client.storage_client.flow_delete(flow_id, skip_trash=True) @@ -154,7 +153,6 @@ async def test_update_flow(mcp_context: Context, configs: list[ConfigDef]) -> No assert metadata_dict[updated_by_md_key] == 'true' finally: - client = KeboolaClient.from_state(mcp_context.session.state) await client.storage_client.flow_delete(flow_id, skip_trash=True)