Skip to content

Commit c2decc3

Browse files
aji-ajuaji-aju
authored
Merge branch 'main' into import/export-functionality-disable
2 parents 649b42d + 80a525b commit c2decc3

682 files changed

Lines changed: 50199 additions & 11973 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/py-cli-e2e-tests.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -182,12 +182,12 @@ jobs:
182182
echo "import os" >> $SITE_CUSTOMIZE_PATH
183183
echo "try:" >> $SITE_CUSTOMIZE_PATH
184184
echo " import coverage" >> $SITE_CUSTOMIZE_PATH
185-
echo " os.environ['COVERAGE_PROCESS_START'] = 'ingestion/pyproject.toml'" >> $SITE_CUSTOMIZE_PATH
185+
echo " os.environ['COVERAGE_PROCESS_START'] = os.path.join(os.environ.get('GITHUB_WORKSPACE', os.getcwd()), 'ingestion', 'pyproject.toml')" >> $SITE_CUSTOMIZE_PATH
186186
echo " coverage.process_startup()" >> $SITE_CUSTOMIZE_PATH
187187
echo "except ImportError:" >> $SITE_CUSTOMIZE_PATH
188188
echo " pass" >> $SITE_CUSTOMIZE_PATH
189-
coverage run --rcfile ingestion/pyproject.toml -a --branch -m pytest -c ingestion/pyproject.toml --junitxml=ingestion/junit/test-results-$E2E_TEST.xml --ignore=ingestion/tests/unit/source ingestion/tests/cli_e2e/test_cli_$E2E_TEST.py
190-
coverage combine --data-file=.coverage.$E2E_TEST --rcfile=ingestion/pyproject.toml --keep -a .coverage*
189+
coverage run --rcfile ingestion/pyproject.toml --branch -m pytest -c ingestion/pyproject.toml --junitxml=ingestion/junit/test-results-$E2E_TEST.xml --ignore=ingestion/tests/unit/source ingestion/tests/cli_e2e/test_cli_$E2E_TEST.py
190+
coverage combine --data-file=.coverage.$E2E_TEST --rcfile=ingestion/pyproject.toml --keep .coverage*
191191
coverage report --rcfile ingestion/pyproject.toml --data-file .coverage.$E2E_TEST || true
192192
193193
- name: Upload coverage artifact for Python unit tests
@@ -293,7 +293,7 @@ jobs:
293293
done
294294
source env/bin/activate
295295
cd ingestion
296-
coverage combine --rcfile=pyproject.toml --keep -a .coverage*
296+
coverage combine --rcfile=pyproject.toml --keep .coverage*
297297
coverage xml --rcfile=pyproject.toml --data-file=.coverage
298298
shell: bash
299299

‎bootstrap/sql/migrations/native/1.13.0/mysql/schemaChanges.sql‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,86 @@ FROM user_entity ue, role_entity re
130130
WHERE ue.name = 'mcpapplicationbot'
131131
AND re.name = 'ApplicationBotImpersonationRole';
132132

133+
134+
UPDATE entity_extension
135+
SET json = JSON_SET(
136+
json,
137+
'$.profileSampleConfig',
138+
JSON_OBJECT(
139+
'sampleConfigType', 'STATIC',
140+
'config', JSON_OBJECT(
141+
'profileSample', JSON_EXTRACT(json, '$.profileSample'),
142+
'profileSampleType', COALESCE(
143+
JSON_EXTRACT(json, '$.profileSampleType'),
144+
CAST('"PERCENTAGE"' AS JSON)
145+
),
146+
'samplingMethodType', JSON_EXTRACT(json, '$.samplingMethodType')
147+
)
148+
)
149+
)
150+
WHERE extension IN (
151+
'table.tableProfilerConfig',
152+
'database.databaseProfilerConfig',
153+
'databaseSchema.databaseSchemaProfilerConfig'
154+
)
155+
AND JSON_EXTRACT(json, '$.profileSample') IS NOT NULL
156+
AND JSON_TYPE(JSON_EXTRACT(json, '$.profileSample')) != 'NULL'
157+
AND NOT JSON_CONTAINS_PATH(json, 'one', '$.profileSampleConfig');
158+
159+
-- entity_extension: remove old flat fields
160+
UPDATE entity_extension
161+
SET json = JSON_REMOVE(
162+
JSON_REMOVE(
163+
JSON_REMOVE(json, '$.samplingMethodType'),
164+
'$.profileSampleType'
165+
),
166+
'$.profileSample'
167+
)
168+
WHERE extension IN (
169+
'table.tableProfilerConfig',
170+
'database.databaseProfilerConfig',
171+
'databaseSchema.databaseSchemaProfilerConfig'
172+
)
173+
AND (JSON_CONTAINS_PATH(json, 'one', '$.profileSample')
174+
OR JSON_CONTAINS_PATH(json, 'one', '$.profileSampleType')
175+
OR JSON_CONTAINS_PATH(json, 'one', '$.samplingMethodType'));
176+
177+
-- ingestion_pipeline_entity (profiler pipelines): build profileSampleConfig (skip if already migrated)
178+
UPDATE ingestion_pipeline_entity
179+
SET json = JSON_SET(
180+
json,
181+
'$.sourceConfig.config.profileSampleConfig',
182+
JSON_OBJECT(
183+
'sampleConfigType', 'STATIC',
184+
'config', JSON_OBJECT(
185+
'profileSample', JSON_EXTRACT(json, '$.sourceConfig.config.profileSample'),
186+
'profileSampleType', COALESCE(
187+
JSON_EXTRACT(json, '$.sourceConfig.config.profileSampleType'),
188+
CAST('"PERCENTAGE"' AS JSON)
189+
),
190+
'samplingMethodType', JSON_EXTRACT(json, '$.sourceConfig.config.samplingMethodType')
191+
)
192+
)
193+
)
194+
WHERE pipelineType = 'profiler'
195+
AND JSON_EXTRACT(json, '$.sourceConfig.config.profileSample') IS NOT NULL
196+
AND JSON_TYPE(JSON_EXTRACT(json, '$.sourceConfig.config.profileSample')) != 'NULL'
197+
AND NOT JSON_CONTAINS_PATH(json, 'one', '$.sourceConfig.config.profileSampleConfig');
198+
199+
-- ingestion_pipeline_entity (profiler pipelines): remove old flat fields
200+
UPDATE ingestion_pipeline_entity
201+
SET json = JSON_REMOVE(
202+
JSON_REMOVE(
203+
JSON_REMOVE(json, '$.sourceConfig.config.samplingMethodType'),
204+
'$.sourceConfig.config.profileSampleType'
205+
),
206+
'$.sourceConfig.config.profileSample'
207+
)
208+
WHERE pipelineType = 'profiler'
209+
AND (JSON_CONTAINS_PATH(json, 'one', '$.sourceConfig.config.profileSample')
210+
OR JSON_CONTAINS_PATH(json, 'one', '$.sourceConfig.config.profileSampleType')
211+
OR JSON_CONTAINS_PATH(json, 'one', '$.sourceConfig.config.samplingMethodType'));
212+
133213
-- RDF distributed indexing state tables
134214
CREATE TABLE IF NOT EXISTS rdf_index_job (
135215
id VARCHAR(36) NOT NULL,

‎bootstrap/sql/migrations/native/1.13.0/postgres/schemaChanges.sql‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,83 @@ WHERE ue.name = 'mcpapplicationbot'
151151
AND re.name = 'ApplicationBotImpersonationRole'
152152
ON CONFLICT DO NOTHING;
153153

154+
-- Migrate profiler sampling config: move flat profileSample/profileSampleType/samplingMethodType
155+
-- into the new profileSampleConfig structure. Default to STATIC since DYNAMIC is new.
156+
157+
-- Profiler configs are stored in entity_extension table, not in entity json columns.
158+
-- Extension keys: table.tableProfilerConfig, database.databaseProfilerConfig, databaseSchema.databaseSchemaProfilerConfig
159+
-- The json column in entity_extension contains the config object directly (flat root-level fields).
160+
161+
-- entity_extension: build profileSampleConfig from existing flat fields (skip if already migrated)
162+
UPDATE entity_extension
163+
SET json = jsonb_set(
164+
json::jsonb,
165+
'{profileSampleConfig}',
166+
jsonb_build_object(
167+
'sampleConfigType', 'STATIC',
168+
'config', jsonb_build_object(
169+
'profileSample', json::jsonb #> '{profileSample}',
170+
'profileSampleType', COALESCE(
171+
json::jsonb #> '{profileSampleType}',
172+
'"PERCENTAGE"'::jsonb
173+
),
174+
'samplingMethodType', json::jsonb #> '{samplingMethodType}'
175+
)
176+
)
177+
)::json
178+
WHERE extension IN (
179+
'table.tableProfilerConfig',
180+
'database.databaseProfilerConfig',
181+
'databaseSchema.databaseSchemaProfilerConfig'
182+
)
183+
AND json::jsonb #>> '{profileSample}' IS NOT NULL
184+
AND json::jsonb #> '{profileSampleConfig}' IS NULL;
185+
186+
-- entity_extension: remove old flat fields
187+
UPDATE entity_extension
188+
SET json = (json::jsonb #- '{profileSample}'
189+
#- '{profileSampleType}'
190+
#- '{samplingMethodType}')::json
191+
WHERE extension IN (
192+
'table.tableProfilerConfig',
193+
'database.databaseProfilerConfig',
194+
'databaseSchema.databaseSchemaProfilerConfig'
195+
)
196+
AND (json::jsonb #>> '{profileSample}' IS NOT NULL
197+
OR json::jsonb #>> '{profileSampleType}' IS NOT NULL
198+
OR json::jsonb #>> '{samplingMethodType}' IS NOT NULL);
199+
200+
-- ingestion_pipeline_entity (profiler pipelines): build profileSampleConfig (skip if already migrated)
201+
UPDATE ingestion_pipeline_entity
202+
SET json = jsonb_set(
203+
json::jsonb,
204+
'{sourceConfig,config,profileSampleConfig}',
205+
jsonb_build_object(
206+
'sampleConfigType', 'STATIC',
207+
'config', jsonb_build_object(
208+
'profileSample', json::jsonb #> '{sourceConfig,config,profileSample}',
209+
'profileSampleType', COALESCE(
210+
json::jsonb #> '{sourceConfig,config,profileSampleType}',
211+
'"PERCENTAGE"'::jsonb
212+
),
213+
'samplingMethodType', json::jsonb #> '{sourceConfig,config,samplingMethodType}'
214+
)
215+
)
216+
)::json
217+
WHERE json #>> '{pipelineType}' = 'profiler'
218+
AND json::jsonb #>> '{sourceConfig,config,profileSample}' IS NOT NULL
219+
AND json::jsonb #> '{sourceConfig,config,profileSampleConfig}' IS NULL;
220+
221+
-- ingestion_pipeline_entity (profiler pipelines): remove old flat fields
222+
UPDATE ingestion_pipeline_entity
223+
SET json = (json::jsonb #- '{sourceConfig,config,profileSample}'
224+
#- '{sourceConfig,config,profileSampleType}'
225+
#- '{sourceConfig,config,samplingMethodType}')::json
226+
WHERE json #>> '{pipelineType}' = 'profiler'
227+
AND (json::jsonb #>> '{sourceConfig,config,profileSample}' IS NOT NULL
228+
OR json::jsonb #>> '{sourceConfig,config,profileSampleType}' IS NOT NULL
229+
OR json::jsonb #>> '{sourceConfig,config,samplingMethodType}' IS NOT NULL);
230+
154231
-- RDF distributed indexing state tables
155232
CREATE TABLE IF NOT EXISTS rdf_index_job (
156233
id VARCHAR(36) NOT NULL,

‎conf/openmetadata.yaml‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,8 @@ server:
121121
# jceProvider: (none)
122122
# validateCerts: true
123123
# validatePeers: true
124-
# supportedProtocols: SSLv3
125-
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
124+
# supportedProtocols: [TLSv1.2, TLSv1.3]
125+
# supportedCipherSuites: [TLS_AES_256_GCM_SHA384, TLS_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]
126126
# allowRenegotiation: true
127127
# endpointIdentificationAlgorithm: (none)
128128

@@ -149,8 +149,8 @@ server:
149149
# jceProvider: (none)
150150
# validateCerts: true
151151
# validatePeers: true
152-
# supportedProtocols: SSLv3
153-
# supportedCipherSuites: TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
152+
# supportedProtocols: [TLSv1.2, TLSv1.3]
153+
# supportedCipherSuites: [TLS_AES_256_GCM_SHA384, TLS_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256]
154154
# allowRenegotiation: true
155155
# endpointIdentificationAlgorithm: (none)
156156

@@ -708,6 +708,15 @@ web:
708708
permission-policy:
709709
enabled: ${WEB_CONF_PERMISSION_POLICY_ENABLED:-false}
710710
option: ${WEB_CONF_PERMISSION_POLICY_OPTION:-""}
711+
cross-origin-embedder-policy:
712+
enabled: ${WEB_CONF_CROSS_ORIGIN_EMBEDDER_POLICY_ENABLED:-false}
713+
option: ${WEB_CONF_CROSS_ORIGIN_EMBEDDER_POLICY_OPTION:-"REQUIRE_CORP"}
714+
cross-origin-resource-policy:
715+
enabled: ${WEB_CONF_CROSS_ORIGIN_RESOURCE_POLICY_ENABLED:-false}
716+
option: ${WEB_CONF_CROSS_ORIGIN_RESOURCE_POLICY_OPTION:-"SAME_ORIGIN"}
717+
cross-origin-opener-policy:
718+
enabled: ${WEB_CONF_CROSS_ORIGIN_OPENER_POLICY_ENABLED:-false}
719+
option: ${WEB_CONF_CROSS_ORIGIN_OPENER_POLICY_OPTION:-"SAME_ORIGIN"}
711720
cache-control: ${WEB_CONF_CACHE_CONTROL:-""}
712721
pragma: ${WEB_CONF_PRAGMA:-""}
713722

‎ingestion/operators/docker/Dockerfile‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,16 @@ RUN if [ $(uname -m) = "arm64" || $(uname -m) = "aarch64" ]; \
6363
ENV LD_LIBRARY_PATH=/instantclient
6464

6565
# Install DB2 iAccess driver
66-
RUN if [ $(uname -m) = "x86_64" ]; \
67-
then \
68-
curl https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list | tee /etc/apt/sources.list.d/ibmi-acs-1.1.0.list \
69-
&& apt update \
70-
&& apt install ibm-iaccess; \
71-
fi
66+
# The IBM repository approach is unreliable in CI environments, so we download the package directly
67+
# Use dpkg --force-depends because the package declares old Debian package names (libodbc1, odbcinst1debian2)
68+
# that don't exist in Debian 12, but the actual dependencies (unixodbc, odbcinst) are already installed
69+
RUN if [ $(uname -m) = "x86_64" ]; then \
70+
wget -q https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/main/binary-amd64/ibm-iaccess-1.1.0.13-1.0.amd64.deb \
71+
-O /tmp/ibm-iaccess.deb && \
72+
dpkg -i --force-depends /tmp/ibm-iaccess.deb && \
73+
apt-get install -f -y --no-install-recommends && \
74+
rm -f /tmp/ibm-iaccess.deb; \
75+
fi
7276

7377
WORKDIR ingestion/
7478

‎ingestion/operators/docker/Dockerfile.ci‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -63,12 +63,16 @@ RUN if [ $(uname -m) = "arm64" ] | [ $(uname -m) = "aarch64" ]; \
6363
ENV LD_LIBRARY_PATH=/instantclient
6464

6565
# Install DB2 iAccess Driver
66-
RUN if [ $(uname -m) = "x86_64" ]; \
67-
then \
68-
curl https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list | tee /etc/apt/sources.list.d/ibmi-acs-1.1.0.list \
69-
&& apt update \
70-
&& apt install ibm-iaccess; \
71-
fi
66+
# The IBM repository approach is unreliable in CI environments, so we download the package directly
67+
# Use dpkg --force-depends because the package declares old Debian package names (libodbc1, odbcinst1debian2)
68+
# that don't exist in Debian 12, but the actual dependencies (unixodbc, odbcinst) are already installed
69+
RUN if [ $(uname -m) = "x86_64" ]; then \
70+
wget -q https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/main/binary-amd64/ibm-iaccess-1.1.0.13-1.0.amd64.deb \
71+
-O /tmp/ibm-iaccess.deb && \
72+
dpkg -i --force-depends /tmp/ibm-iaccess.deb && \
73+
apt-get install -f -y --no-install-recommends && \
74+
rm -f /tmp/ibm-iaccess.deb; \
75+
fi
7276

7377
WORKDIR /ingestion
7478

‎ingestion/pyproject.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ provider_info = "airflow_provider_openmetadata:get_provider_config"
4343
source = ["metadata"]
4444
relative_files = true
4545
branch = true
46+
parallel = true
4647

4748
# Remap installed-package paths back to the source tree so that
4849
# ``coverage combine`` (in a lightweight CI job without the package

‎ingestion/setup.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# https://github.com/open-metadata/OpenMetadata/actions/runs/15640676139/job/44066998708?pr=21719 Copyright 2025 Collate
1+
# Copyright 2025 Collate
22
# Licensed under the Collate Community License, Version 1.0 (the "License");
33
# you may not use this file except in compliance with the License.
44
# You may obtain a copy of the License at
@@ -192,7 +192,12 @@
192192
"atlas": {},
193193
"azuresql": {VERSIONS["pyodbc"]},
194194
"azure-sso": {VERSIONS["msal"]},
195+
"microsoftfabric": {VERSIONS["pyodbc"], VERSIONS["msal"]},
196+
"microsoftfabricpipeline": {VERSIONS["msal"]},
195197
"backup": {VERSIONS["boto3"], VERSIONS["azure-identity"], "azure-storage-blob"},
198+
"googledrive": {
199+
"google-api-python-client>=2.0.0",
200+
},
196201
"bigquery": {
197202
"google-cloud-datacatalog>=3.6.2",
198203
"google-cloud-logging",

‎ingestion/src/metadata/clients/microsoftfabric/__init__.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Copyright 2025 Collate
2+
# Licensed under the Collate Community License, Version 1.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
# https://github.com/open-metadata/OpenMetadata/blob/main/ingestion/LICENSE
6+
# Unless required by applicable law or agreed to in writing, software
7+
# distributed under the License is distributed on an "AS IS" BASIS,
8+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9+
# See the License for the specific language governing permissions and
10+
# limitations under the License.
11+
"""
12+
Microsoft Fabric client module
13+
"""

0 commit comments

Comments
 (0)