From 1b56035b1d43fbc54b02a2cec35574915c41b5f4 Mon Sep 17 00:00:00 2001 From: amit-apptware <132869468+amit-apptware@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:14:02 +0530 Subject: [PATCH] feat(ui/data-contract): Data contract UI under Validation Tab (#10625) Co-authored-by: jayacryl <159848059+jayacryl@users.noreply.github.com> --- .../graphql/featureflags/FeatureFlags.java | 1 + .../resolvers/config/AppConfigResolver.java | 1 + .../src/main/resources/app.graphql | 5 + .../src/main/resources/entity.graphql | 15 + .../tabs/Dataset/Validations/Assertions.tsx | 15 +- .../Validations/DatasetAssertionsList.tsx | 190 ++++++--- .../Validations/FieldAssertionDescription.tsx | 31 ++ .../Validations/SqlAssertionDescription.tsx | 16 + .../Dataset/Validations/ValidationsTab.tsx | 22 +- .../VolumeAssertionDescription.tsx | 41 ++ .../Dataset/Validations/assertionUtils.tsx | 2 +- .../contract/DataContractAssertionStatus.tsx | 44 ++ .../contract/DataContractEmptyState.tsx | 87 ++++ .../contract/DataContractSummary.tsx | 95 +++++ .../contract/DataContractSummaryFooter.tsx | 75 ++++ .../Validations/contract/DataContractTab.tsx | 117 ++++++ .../contract/DataQualityContractSummary.tsx | 100 +++++ .../contract/FreshnessContractSummary.tsx | 96 +++++ .../contract/FreshnessScheduleSummary.tsx | 28 ++ .../contract/SchemaContractSummary.tsx | 82 ++++ .../DataContractAssertionGroupSelect.tsx | 54 +++ .../contract/builder/DataContractBuilder.tsx | 161 ++++++++ .../builder/DataContractBuilderModal.tsx | 67 ++++ .../Validations/contract/builder/types.ts | 38 ++ .../Validations/contract/builder/utils.ts | 120 ++++++ .../Dataset/Validations/contract/utils.tsx | 114 ++++++ .../Validations/fieldDescriptionUtils.ts | 179 +++++++++ .../Validations/shared/styledComponents.tsx | 32 ++ .../shared/tabs/Dataset/Validations/types.tsx | 21 + .../shared/tabs/Dataset/Validations/utils.tsx | 378 ++++++++++++++++++ .../src/app/shared/numberUtil.ts | 24 ++ datahub-web-react/src/appConfigContext.tsx | 1 + datahub-web-react/src/graphql/app.graphql | 1 + .../src/graphql/assertion.graphql | 12 + .../src/graphql/contract.graphql | 53 +++ .../src/main/resources/application.yaml | 3 +- 36 files changed, 2272 insertions(+), 49 deletions(-) create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/FieldAssertionDescription.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/SqlAssertionDescription.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/VolumeAssertionDescription.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractAssertionStatus.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractEmptyState.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummary.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummaryFooter.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractTab.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataQualityContractSummary.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessContractSummary.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/SchemaContractSummary.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractAssertionGroupSelect.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilder.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilderModal.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/types.ts create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/utils.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/fieldDescriptionUtils.ts create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/shared/styledComponents.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/types.tsx create mode 100644 datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/utils.tsx create mode 100644 datahub-web-react/src/app/shared/numberUtil.ts create mode 100644 datahub-web-react/src/graphql/contract.graphql diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java index 2a4f75cf6055a..2a9af37a06ad9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/featureflags/FeatureFlags.java @@ -20,4 +20,5 @@ public class FeatureFlags { private boolean nestedDomainsEnabled = false; private boolean schemaFieldEntityFetchEnabled = false; private boolean businessAttributeEntityEnabled = false; + private boolean dataContractsEnabled = false; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index c05009e146308..caa469003c22e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -185,6 +185,7 @@ public CompletableFuture get(final DataFetchingEnvironment environmen .setShowAccessManagement(_featureFlags.isShowAccessManagement()) .setNestedDomainsEnabled(_featureFlags.isNestedDomainsEnabled()) .setPlatformBrowseV2(_featureFlags.isPlatformBrowseV2()) + .setDataContractsEnabled(_featureFlags.isDataContractsEnabled()) .build(); appConfig.setFeatureFlags(featureFlagsConfig); diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index d84a86a3bedd3..b3a965981c366 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -492,6 +492,11 @@ type FeatureFlagsConfig { Whether business attribute entity should be shown """ businessAttributeEntityEnabled: Boolean! + + """ + Whether data contracts should be enabled + """ + dataContractsEnabled: Boolean! } """ diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index fa774d34ed7a4..211ae97910202 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -12586,3 +12586,18 @@ type ListBusinessAttributesResult { """ businessAttributes: [BusinessAttribute!]! } + +""" +A cron schedule +""" +type CronSchedule { + """ + A cron-formatted execution interval, as a cron string, e.g. 1 * * * * + """ + cron: String! + + """ + Timezone in which the cron interval applies, e.g. America/Los_Angeles + """ + timezone: String! +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/Assertions.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/Assertions.tsx index b3086d7867012..953433c6a8165 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/Assertions.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/Assertions.tsx @@ -6,6 +6,7 @@ import { DatasetAssertionsList } from './DatasetAssertionsList'; import { DatasetAssertionsSummary } from './DatasetAssertionsSummary'; import { sortAssertions } from './assertionUtils'; import { combineEntityDataWithSiblings, useIsSeparateSiblingsMode } from '../../../siblingUtils'; +import { useGetDatasetContractQuery } from '../../../../../../graphql/contract.generated'; /** * Returns a status summary for the assertions associated with a Dataset. @@ -15,6 +16,7 @@ const getAssertionsStatusSummary = (assertions: Array) => { failedRuns: 0, succeededRuns: 0, totalRuns: 0, + erroredRuns: 0, totalAssertions: assertions.length, }; assertions.forEach((assertion) => { @@ -27,7 +29,12 @@ const getAssertionsStatusSummary = (assertions: Array) => { if (AssertionResultType.Failure === resultType) { summary.failedRuns++; } - summary.totalRuns++; // only count assertions for which there is one completed run event! + if (AssertionResultType.Error === resultType) { + summary.erroredRuns++; + } + if (AssertionResultType.Init !== resultType) { + summary.totalRuns++; // only count assertions for which there is one completed run event, ignoring INIT statuses! + } } }); return summary; @@ -46,6 +53,11 @@ export const Assertions = () => { const combinedData = isHideSiblingMode ? data : combineEntityDataWithSiblings(data); const [removedUrns, setRemovedUrns] = useState([]); + const { data: contractData } = useGetDatasetContractQuery({ + variables: { urn }, + fetchPolicy: 'cache-first', + }); + const contract = contractData?.dataset?.contract as any; const assertions = (combinedData && combinedData.dataset?.assertions?.assertions?.map((assertion) => assertion as Assertion)) || []; @@ -67,6 +79,7 @@ export const Assertions = () => { setRemovedUrns([...removedUrns, assertionUrn]); setTimeout(() => refetch(), 3000); }} + contract={contract} /> )} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx index 3eccfb8931fc0..79f337a8a13f8 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/DatasetAssertionsList.tsx @@ -1,15 +1,27 @@ -import { Button, Dropdown, Empty, Image, message, Modal, Tag, Tooltip, Typography } from 'antd'; +import { Button, Dropdown, Empty, Image, message, Modal, Tag, Tooltip, Typography, Checkbox } from 'antd'; import React from 'react'; import styled from 'styled-components'; -import { DeleteOutlined, DownOutlined, MoreOutlined, RightOutlined, StopOutlined } from '@ant-design/icons'; +import { + DeleteOutlined, + DownOutlined, + MoreOutlined, + RightOutlined, + StopOutlined, + AuditOutlined, +} from '@ant-design/icons'; +import { Link } from 'react-router-dom'; import { DatasetAssertionDescription } from './DatasetAssertionDescription'; import { StyledTable } from '../../../components/styled/StyledTable'; import { DatasetAssertionDetails } from './DatasetAssertionDetails'; -import { Assertion, AssertionRunStatus } from '../../../../../../types.generated'; +import { Assertion, AssertionRunStatus, DataContract, EntityType } from '../../../../../../types.generated'; import { getResultColor, getResultIcon, getResultText } from './assertionUtils'; import { useDeleteAssertionMutation } from '../../../../../../graphql/assertion.generated'; import { capitalizeFirstLetterOnly } from '../../../../../shared/textUtil'; import AssertionMenu from './AssertionMenu'; +import { REDESIGN_COLORS } from '../../../constants'; +import { useEntityRegistry } from '../../../../../useEntityRegistry'; +import { isAssertionPartOfContract } from './contract/utils'; +import { useEntityData } from '../../../EntityContext'; const ResultContainer = styled.div` display: flex; @@ -35,9 +47,26 @@ const StyledMoreOutlined = styled(MoreOutlined)` font-size: 18px; `; +const AssertionSelectCheckbox = styled(Checkbox)` + margin-right: 12px; +`; + +const DataContractLogo = styled(AuditOutlined)` + margin-left: 8px; + font-size: 16px; + color: ${REDESIGN_COLORS.BLUE}; +`; + type Props = { assertions: Array; onDelete?: (urn: string) => void; + contract?: DataContract; + // required for enabling menu/actions + showMenu?: boolean; + onSelect?: (assertionUrn: string) => void; + // required for enabling selection logic + showSelect?: boolean; + selectedUrns?: string[]; }; /** @@ -46,8 +75,18 @@ type Props = { * * Currently this component supports rendering Dataset Assertions only. */ -export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => { +export const DatasetAssertionsList = ({ + assertions, + onDelete, + showMenu = true, + showSelect, + onSelect, + selectedUrns, + contract, +}: Props) => { + const entityData = useEntityData(); const [deleteAssertionMutation] = useDeleteAssertionMutation(); + const entityRegistry = useEntityRegistry(); const deleteAssertion = async (urn: string) => { try { @@ -102,9 +141,19 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => { const resultColor = (record.lastExecResult && getResultColor(record.lastExecResult)) || 'default'; const resultText = (record.lastExecResult && getResultText(record.lastExecResult)) || 'No Evaluations'; const resultIcon = (record.lastExecResult && getResultIcon(record.lastExecResult)) || ; + const selected = selectedUrns?.some((selectedUrn) => selectedUrn === record.urn); + const isPartOfContract = contract && isAssertionPartOfContract(record, contract); + const { description } = record; return ( + {showSelect ? ( + e.stopPropagation()} + onChange={() => onSelect?.(record.urn as string)} + /> + ) : undefined}
@@ -117,6 +166,34 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => { description={description} assertionInfo={record.datasetAssertionInfo} /> + {(isPartOfContract && entityData?.urn && ( + + Part of Data Contract{' '} + + view + + + } + > + + + + + )) || + undefined} ); }, @@ -126,35 +203,40 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => { dataIndex: '', key: '', render: (_, record: any) => ( - - - - {(record.platform.properties?.logoUrl && ( - - )) || ( - - {record.platform.properties?.displayName || - capitalizeFirstLetterOnly(record.platform.name)} - - )} - - - - } trigger={['click']}> - - - + <> + {showMenu && ( + + + + {(record.platform.properties?.logoUrl && ( + + )) || ( + + {record.platform.properties?.displayName || + capitalizeFirstLetterOnly(record.platform.name)} + + )} + + + + } trigger={['click']}> + + + + )} + ), }, ]; @@ -168,18 +250,36 @@ export const DatasetAssertionsList = ({ assertions, onDelete }: Props) => { locale={{ emptyText: , }} - expandable={{ - defaultExpandAllRows: false, - expandRowByClick: true, - expandedRowRender: (record) => { - return ; - }, - expandIcon: ({ expanded, onExpand, record }: any) => - expanded ? ( - onExpand(record, e)} /> - ) : ( - onExpand(record, e)} /> - ), + expandable={ + showSelect + ? {} + : { + defaultExpandAllRows: false, + expandRowByClick: true, + expandedRowRender: (record) => { + return ( + + ); + }, + expandIcon: ({ expanded, onExpand, record }: any) => + expanded ? ( + onExpand(record, e)} /> + ) : ( + onExpand(record, e)} /> + ), + } + } + onRow={(record) => { + return { + onClick: (_) => { + if (showSelect) { + onSelect?.(record.urn as string); + } + }, + }; }} showHeader={false} pagination={false} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/FieldAssertionDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/FieldAssertionDescription.tsx new file mode 100644 index 0000000000000..a104903dc7bc2 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/FieldAssertionDescription.tsx @@ -0,0 +1,31 @@ +import React from 'react'; +import { Typography } from 'antd'; +import { FieldAssertionInfo } from '../../../../../../types.generated'; +import { + getFieldDescription, + getFieldOperatorDescription, + getFieldParametersDescription, + getFieldTransformDescription, +} from './fieldDescriptionUtils'; + +type Props = { + assertionInfo: FieldAssertionInfo; +}; + +/** + * A human-readable description of a Field Assertion. + */ +export const FieldAssertionDescription = ({ assertionInfo }: Props) => { + const field = getFieldDescription(assertionInfo); + const operator = getFieldOperatorDescription(assertionInfo); + const transform = getFieldTransformDescription(assertionInfo); + const parameters = getFieldParametersDescription(assertionInfo); + + return ( + + {transform} + {transform ? ' of ' : ''} + {field} {operator} {parameters} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/SqlAssertionDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/SqlAssertionDescription.tsx new file mode 100644 index 0000000000000..047f7c7db28f6 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/SqlAssertionDescription.tsx @@ -0,0 +1,16 @@ +import React from 'react'; +import { Typography } from 'antd'; +import { AssertionInfo } from '../../../../../../types.generated'; + +type Props = { + assertionInfo: AssertionInfo; +}; + +/** + * A human-readable description of a SQL Assertion. + */ +export const SqlAssertionDescription = ({ assertionInfo }: Props) => { + const { description } = assertionInfo; + + return {description}; +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx index b4f77196edbb1..92af9bfc2b567 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/ValidationsTab.tsx @@ -2,13 +2,15 @@ import React, { useEffect } from 'react'; import { Button } from 'antd'; import { useHistory, useLocation } from 'react-router'; import styled from 'styled-components'; -import { FileDoneOutlined, FileProtectOutlined } from '@ant-design/icons'; +import { AuditOutlined, FileDoneOutlined, FileProtectOutlined } from '@ant-design/icons'; import { useEntityData } from '../../../EntityContext'; import { TestResults } from './TestResults'; import { Assertions } from './Assertions'; import TabToolbar from '../../../components/styled/TabToolbar'; import { useGetValidationsTab } from './useGetValidationsTab'; import { ANTD_GRAY } from '../../../constants'; +import { useAppConfig } from '../../../../../useAppConfig'; +import { DataContractTab } from './contract/DataContractTab'; const TabTitle = styled.span` margin-left: 4px; @@ -22,6 +24,7 @@ const TabButton = styled(Button)<{ selected: boolean }>` enum TabPaths { ASSERTIONS = 'Assertions', TESTS = 'Tests', + DATA_CONTRACT = 'Data Contract', } const DEFAULT_TAB = TabPaths.ASSERTIONS; @@ -33,6 +36,7 @@ export const ValidationsTab = () => { const { entityData } = useEntityData(); const history = useHistory(); const { pathname } = useLocation(); + const appConfig = useAppConfig(); const totalAssertions = (entityData as any)?.assertions?.total; const passingTests = (entityData as any)?.testResults?.passing || []; @@ -77,6 +81,22 @@ export const ValidationsTab = () => { }, ]; + if (appConfig.config.featureFlags?.dataContractsEnabled) { + // If contracts feature is enabled, add to list. + tabs.push({ + title: ( + <> + + + Data Contract + + ), + path: TabPaths.DATA_CONTRACT, + content: , + disabled: false, + }); + } + return ( <> diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/VolumeAssertionDescription.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/VolumeAssertionDescription.tsx new file mode 100644 index 0000000000000..26634c459df0d --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/VolumeAssertionDescription.tsx @@ -0,0 +1,41 @@ +import React from 'react'; +import { Typography } from 'antd'; +import { + IncrementingSegmentRowCountChange, + RowCountChange, + VolumeAssertionInfo, +} from '../../../../../../types.generated'; +import { + getIsRowCountChange, + getOperatorDescription, + getParameterDescription, + getValueChangeTypeDescription, + getVolumeTypeDescription, + getVolumeTypeInfo, +} from './utils'; + +type Props = { + assertionInfo: VolumeAssertionInfo; +}; + +/** + * A human-readable description of a Volume Assertion. + */ +export const VolumeAssertionDescription = ({ assertionInfo }: Props) => { + const volumeType = assertionInfo.type; + const volumeTypeInfo = getVolumeTypeInfo(assertionInfo); + const volumeTypeDescription = getVolumeTypeDescription(volumeType); + const operatorDescription = volumeTypeInfo ? getOperatorDescription(volumeTypeInfo.operator) : ''; + const parameterDescription = volumeTypeInfo ? getParameterDescription(volumeTypeInfo.parameters) : ''; + const valueChangeTypeDescription = getIsRowCountChange(volumeType) + ? getValueChangeTypeDescription((volumeTypeInfo as RowCountChange | IncrementingSegmentRowCountChange).type) + : 'rows'; + + return ( +
+ + Table {volumeTypeDescription} {operatorDescription} {parameterDescription} {valueChangeTypeDescription} + +
+ ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/assertionUtils.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/assertionUtils.tsx index 1eaacb36515a1..341742f407f73 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/assertionUtils.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/assertionUtils.tsx @@ -18,7 +18,7 @@ export const sortAssertions = (a, b) => { if (!b.runEvents?.runEvents?.length) { return -1; } - return b.runEvents.runEvents[0].timestampMillis - a.runEvents.runEvents[0].timestampMillis; + return b.runEvents?.runEvents[0]?.timestampMillis - a.runEvents?.runEvents[0]?.timestampMillis; }; /** diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractAssertionStatus.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractAssertionStatus.tsx new file mode 100644 index 0000000000000..c36bef09cdb68 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractAssertionStatus.tsx @@ -0,0 +1,44 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Tooltip } from 'antd'; +import { StopOutlined } from '@ant-design/icons'; +import { Assertion, AssertionResultType } from '../../../../../../../types.generated'; +import { + StyledCheckOutlined, + StyledClockCircleOutlined, + StyledCloseOutlined, + StyledExclamationOutlined, +} from '../shared/styledComponents'; + +const StatusContainer = styled.div` + width: 100%; + display: flex; + justify-content: center; +`; + +type Props = { + assertion: Assertion; +}; + +export const DataContractAssertionStatus = ({ assertion }: Props) => { + const latestRun = (assertion.runEvents?.runEvents?.length && assertion.runEvents?.runEvents[0]) || undefined; + const latestResultType = latestRun?.result?.type || undefined; + + return ( + + {latestResultType === undefined && } + + {latestResultType === AssertionResultType.Success && } + + + {latestResultType === AssertionResultType.Failure && } + + + {latestResultType === AssertionResultType.Error && } + + + {latestResultType === AssertionResultType.Init && } + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractEmptyState.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractEmptyState.tsx new file mode 100644 index 0000000000000..59a9d7a56821a --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractEmptyState.tsx @@ -0,0 +1,87 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Button, Typography } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; + +import { ANTD_GRAY } from '../../../../constants'; + +const Container = styled.div``; + +const Summary = styled.div` + width: 100%; + padding-left: 40px; + padding-top: 20px; + padding-bottom: 20px; + display: flex; + align-items: center; + justify-content: space-between; + border-bottom: 1px solid ${ANTD_GRAY[4.5]}; + box-shadow: 0px 2px 6px 0px #0000000d; +`; + +const SummaryDescription = styled.div` + display: flex; + align-items: center; +`; + +const SummaryMessage = styled.div` + display: inline-block; + margin-left: 20px; + max-width: 350px; +`; + +const SummaryTitle = styled(Typography.Title)` + && { + padding-bottom: 0px; + margin-bottom: 4px; + } +`; + +const Actions = styled.div` + margin: 12px; + margin-right: 20px; +`; + +const CreateButton = styled(Button)` + margin-right: 12px; + border-color: ${(props) => props.theme.styles['primary-color']}; + color: ${(props) => props.theme.styles['primary-color']}; + letter-spacing: 2px; + &&:hover { + color: white; + background-color: ${(props) => props.theme.styles['primary-color']}; + border-color: ${(props) => props.theme.styles['primary-color']}; + } +`; + +type Props = { + showContractBuilder: () => void; +}; + + +export const DataContractEmptyState = ({ showContractBuilder }: Props) => { + return ( + + + + + + No contract found +
+ + A contract does not yet exist for this dataset + +
+ + + + + + + CREATE + + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummary.tsx new file mode 100644 index 0000000000000..9b684486cb5ce --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummary.tsx @@ -0,0 +1,95 @@ +import React from 'react'; +import styled from 'styled-components'; +import { EditFilled } from '@ant-design/icons'; +import { Button, Typography } from 'antd'; +import { DataContractState } from '../../../../../../../types.generated'; +import { AssertionStatusSummary } from '../types'; +import { getContractSummaryIcon, getContractSummaryTitle, getContractSummaryMessage } from './utils'; +import { ANTD_GRAY } from '../../../../constants'; + +const SummaryHeader = styled.div` + width: 100%; + padding-left: 40px; + padding-top: 20px; + padding-bottom: 20px; + display: flex; + align-items: center; + justify-content: space-between; + border-bottom: 1px solid ${ANTD_GRAY[4.5]}; +`; + +const SummaryContainer = styled.div``; + +const SummaryDescription = styled.div` + display: flex; + align-items: center; +`; + +const SummaryMessage = styled.div` + display: inline-block; + margin-left: 20px; +`; + +const SummaryTitle = styled(Typography.Title)` + && { + padding-bottom: 0px; + margin-bottom: 0px; + } +`; + +const Actions = styled.div` + margin: 12px; + margin-right: 20px; +`; + +const CreateButton = styled(Button)` + display: flex; + align-items: center; + gap: 0.3rem; + margin-right: 12px; + border-color: ${(props) => props.theme.styles['primary-color']}; + color: ${(props) => props.theme.styles['primary-color']}; + letter-spacing: 2px; + &&:hover { + color: white; + background-color: ${(props) => props.theme.styles['primary-color']}; + border-color: ${(props) => props.theme.styles['primary-color']}; + } +`; + +const EditIconStyle = styled(EditFilled)` + && { + font-size: 12px; + } +`; + +type Props = { + state: DataContractState; + summary: AssertionStatusSummary; + showContractBuilder: () => void; +}; + +export const DataContractSummary = ({ state, summary, showContractBuilder }: Props) => { + const summaryIcon = getContractSummaryIcon(state, summary); + const summaryTitle = getContractSummaryTitle(state, summary); + const summaryMessage = getContractSummaryMessage(state, summary); + return ( + + + + {summaryIcon} + + {summaryTitle} + {summaryMessage} + + + + + + + EDIT + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummaryFooter.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummaryFooter.tsx new file mode 100644 index 0000000000000..6a892ebe2417a --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractSummaryFooter.tsx @@ -0,0 +1,75 @@ +import React from 'react'; +import styled from 'styled-components'; +import { ArrowRightOutlined } from '@ant-design/icons'; +import { Button } from 'antd'; +import { Assertion } from '../../../../../../../types.generated'; +import { StyledCheckOutlined, StyledCloseOutlined, StyledExclamationOutlined } from '../shared/styledComponents'; +import { getAssertionsSummary } from '../utils'; +import { ANTD_GRAY, REDESIGN_COLORS } from '../../../../constants'; + +const Container = styled.div` + display: flex; + align-items: center; + justify-content: space-between; +`; + +const StatusContainer = styled.div` + display: flex; + align-items: center; +`; + +const StatusText = styled.div` + color: ${ANTD_GRAY[8]}; + margin-left: 4px; +`; + +const ActionButton = styled(Button)` + color: ${REDESIGN_COLORS.BLUE}; +`; + +const StyledArrowRightOutlined = styled(ArrowRightOutlined)` + font-size: 8px; +`; + +type Props = { + assertions: Assertion[]; + passingText: string; + failingText: string; + errorText: string; + actionText?: string; + showAction?: boolean; +}; + +export const DataContractSummaryFooter = ({ + assertions, + actionText, + passingText, + errorText, + failingText, + showAction = true, +}: Props) => { + const summary = getAssertionsSummary(assertions); + const isFailing = summary.failing > 0; + const isPassing = summary.passing && summary.passing === summary.total; + const isErroring = summary.erroring > 0; + return ( + + + {(isFailing && ) || undefined} + {(isPassing && ) || undefined} + {(isErroring && !isFailing && ) || undefined} + + {(isFailing && failingText) || undefined} + {(isPassing && passingText) || undefined} + {(isErroring && errorText) || undefined} + + + {showAction && ( + + {actionText} + + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractTab.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractTab.tsx new file mode 100644 index 0000000000000..52a7eca8730be --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataContractTab.tsx @@ -0,0 +1,117 @@ +import React, { useState } from 'react'; +import styled from 'styled-components'; +import { useGetDatasetContractQuery } from '../../../../../../../graphql/contract.generated'; +import { DataContractState } from '../../../../../../../types.generated'; +import { useEntityData } from '../../../../EntityContext'; +import { DataContractEmptyState } from './DataContractEmptyState'; +import { DataContractSummary } from './DataContractSummary'; +import { DataQualityContractSummary } from './DataQualityContractSummary'; +import { SchemaContractSummary } from './SchemaContractSummary'; +import { FreshnessContractSummary } from './FreshnessContractSummary'; +import { DataContractBuilderModal } from './builder/DataContractBuilderModal'; +import { createBuilderState } from './builder/utils'; +import { getAssertionsSummary } from '../utils'; + +const Container = styled.div` + display: flex; +`; + +const LeftColumn = styled.div` + width: 50%; +`; + +const RightColumn = styled.div` + width: 50%; +`; + +/** + * Component used for rendering the Data Contract Tab on the Assertions parent tab. + */ +export const DataContractTab = () => { + const { urn } = useEntityData(); + + const { data, refetch } = useGetDatasetContractQuery({ + variables: { + urn, + }, + }); + const [showContractBuilder, setShowContractBuilder] = useState(false); + + const contract = data?.dataset?.contract; + const schemaContracts = data?.dataset?.contract?.properties?.schema || []; + const freshnessContracts = data?.dataset?.contract?.properties?.freshness || []; + const dataQualityContracts = data?.dataset?.contract?.properties?.dataQuality || []; + const schemaAssertions = schemaContracts.map((c) => c.assertion); + const freshnessAssertions = freshnessContracts.map((c) => c.assertion); + const dataQualityAssertions = dataQualityContracts.map((c) => c.assertion); + const assertionsSummary = getAssertionsSummary([ + ...schemaAssertions, + ...freshnessAssertions, + ...dataQualityAssertions, + ] as any); + const contractState = data?.dataset?.contract?.status?.state || DataContractState.Active; + const hasFreshnessContract = freshnessContracts && freshnessContracts?.length; + const hasSchemaContract = schemaContracts && schemaContracts?.length; + const hasDataQualityContract = dataQualityContracts && dataQualityContracts?.length; + const showLeftColumn = hasFreshnessContract || hasSchemaContract || undefined; + + const onContractUpdate = () => { + if (contract) { + // Contract exists, just refetch. + refetch(); + } else { + // no contract yet, wait for indxing, + setTimeout(() => refetch(), 3000); + } + setShowContractBuilder(false); + }; + + return ( + <> + {data?.dataset?.contract ? ( + <> + setShowContractBuilder(true)} + /> + + {showLeftColumn && ( + + {(hasFreshnessContract && ( + + )) || + undefined} + {(hasSchemaContract && ( + + )) || + undefined} + + )} + + {hasDataQualityContract ? ( + + ) : undefined} + + + + ) : ( + setShowContractBuilder(true)} /> + )} + {showContractBuilder && ( + setShowContractBuilder(false)} + onSubmit={onContractUpdate} + /> + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataQualityContractSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataQualityContractSummary.tsx new file mode 100644 index 0000000000000..5b01540f859c6 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/DataQualityContractSummary.tsx @@ -0,0 +1,100 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Table } from 'antd'; +import { Assertion, DataQualityContract, DatasetAssertionInfo } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { DataContractAssertionStatus } from './DataContractAssertionStatus'; +import { DataContractSummaryFooter } from './DataContractSummaryFooter'; +import { DatasetAssertionDescription } from '../DatasetAssertionDescription'; +import { FieldAssertionDescription } from '../FieldAssertionDescription'; +import { SqlAssertionDescription } from '../SqlAssertionDescription'; +import { VolumeAssertionDescription } from '../VolumeAssertionDescription'; + +const TitleText = styled.div` + color: ${ANTD_GRAY[7]}; + margin-bottom: 20px; + letter-spacing: 1px; +`; + +const ColumnHeader = styled.div` + color: ${ANTD_GRAY[8]}; +`; + +const Container = styled.div` + padding: 28px; +`; + +const SummaryContainer = styled.div` + width: 100%; + display: flex; + align-items: center; +`; + +const StyledTable = styled(Table)` + width: 100%; + border-radius: 8px; + box-shadow: 0px 0px 4px rgba(0, 0, 0, 0.1); +`; + +type Props = { + contracts: DataQualityContract[]; + showAction?: boolean; +}; + +export const DataQualityContractSummary = ({ contracts, showAction = false }: Props) => { + const assertions: Assertion[] = contracts?.map((contract) => contract.assertion); + + const columns = [ + { + title: () => ASSERTION, + render: (assertion: Assertion) => ( + <> + {assertion.info?.datasetAssertion && ( + + )} + {assertion.info?.volumeAssertion && ( + + )} + {assertion.info?.fieldAssertion && ( + + )} + {assertion.info?.sqlAssertion && } + + ), + }, + { + title: () => STATUS, + render: (assertion: Assertion) => , + }, + ]; + + const data = (assertions || []).map((assertion) => ({ + ...assertion, + key: assertion.urn, + })); + + return ( + + DATA QUALITY + + ( + + )} + /> + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessContractSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessContractSummary.tsx new file mode 100644 index 0000000000000..efd0151b69bc2 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessContractSummary.tsx @@ -0,0 +1,96 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Divider } from 'antd'; +import { ClockCircleOutlined } from '@ant-design/icons'; +import { FreshnessContract } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { DataContractSummaryFooter } from './DataContractSummaryFooter'; +import { FreshnessScheduleSummary } from './FreshnessScheduleSummary'; + +const Container = styled.div` + padding: 28px; +`; + +const TitleText = styled.div` + color: ${ANTD_GRAY[7]}; + margin-bottom: 20px; + letter-spacing: 1px; +`; + +const ThinDivider = styled(Divider)` + && { + padding: 0px; + margin: 0px; + } +`; + +const Header = styled.div` + color: ${ANTD_GRAY[8]}; + letter-spacing; 4px; + padding-top: 8px; + padding: 12px; + background-color: ${ANTD_GRAY[2]}; +`; + +const Body = styled.div` + padding: 12px; +`; + +const Footer = styled.div` + padding-top: 8px; + padding: 12px; + background-color: ${ANTD_GRAY[2]}; +`; + +const SummaryContainer = styled.div` + width: 100%; + border-radius: 8px; + box-shadow: 0px 0px 4px rgba(0, 0, 0, 0.1); +`; + +type Props = { + contracts: FreshnessContract[]; + showAction?: boolean; +}; + +export const FreshnessContractSummary = ({ contracts, showAction = false }: Props) => { + // TODO: Support multiple per-asset contracts. + const firstContract = (contracts.length && contracts[0]) || undefined; + const assertionDefinition = firstContract?.assertion?.info?.freshnessAssertion?.schedule; + const evaluationSchedule = (firstContract?.assertion as any)?.monitor?.relationships[0]?.entity?.info + ?.assertionMonitor?.assertions[0]?.schedule; + + return ( + + FRESHNESS + +
+ + UPDATE FREQUENCY +
+ + {!assertionDefinition && <>No contract found :(} + + {assertionDefinition && ( + + )} + + + +
+ +
+ + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx new file mode 100644 index 0000000000000..434ccb985574f --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/FreshnessScheduleSummary.tsx @@ -0,0 +1,28 @@ +import React from 'react'; +import cronstrue from 'cronstrue'; +import { + FreshnessAssertionSchedule, + FreshnessAssertionScheduleType, + CronSchedule, +} from '../../../../../../../types.generated'; +import { capitalizeFirstLetter } from '../../../../../../shared/textUtil'; + +type Props = { + definition: FreshnessAssertionSchedule; + evaluationSchedule?: CronSchedule; // When the assertion is run. +}; + +export const FreshnessScheduleSummary = ({ definition, evaluationSchedule }: Props) => { + const scheduleText = + definition.type === FreshnessAssertionScheduleType.Cron + ? `${capitalizeFirstLetter(cronstrue.toString(definition.cron?.cron as string))}.` + : `In the past ${ + definition.fixedInterval?.multiple + } ${definition.fixedInterval?.unit.toLocaleLowerCase()}s${ + (evaluationSchedule && + `, as of ${cronstrue.toString(evaluationSchedule.cron as string).toLowerCase()}`) || + '' + }`; + + return <>{scheduleText}; +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/SchemaContractSummary.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/SchemaContractSummary.tsx new file mode 100644 index 0000000000000..7313a1064634c --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/SchemaContractSummary.tsx @@ -0,0 +1,82 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Table } from 'antd'; +import { SchemaContract } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { DataContractSummaryFooter } from './DataContractSummaryFooter'; + +const TitleText = styled.div` + color: ${ANTD_GRAY[7]}; + margin-bottom: 20px; + letter-spacing: 1px; +`; + +const ColumnHeader = styled.div` + color: ${ANTD_GRAY[8]}; +`; + +const Container = styled.div` + padding: 28px; +`; + +const SummaryContainer = styled.div` + width: 100%; + display: flex; + align-items: center; +`; + +const StyledTable = styled(Table)` + width: 100%; + border-radius: 8px; + box-shadow: 0px 0px 4px rgba(0, 0, 0, 0.1); + height: 100%; +`; + +type Props = { + contracts: SchemaContract[]; + showAction?: boolean; +}; + +export const SchemaContractSummary = ({ contracts, showAction = false }: Props) => { + const firstContract = (contracts.length && contracts[0]) || undefined; + const schemaMetadata = firstContract?.assertion?.info?.schemaAssertion?.schema; + + const columns = [ + { + title: () => NAME, + render: (field) => <>{field.fieldPath}, + }, + { + title: () => TYPE, + render: (field) => <>{field.type}, + }, + ]; + + const data = (schemaMetadata?.fields || []).map((field) => ({ + ...field, + key: field.fieldPath, + })); + + return ( + + SCHEMA + + ( + + )} + /> + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractAssertionGroupSelect.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractAssertionGroupSelect.tsx new file mode 100644 index 0000000000000..f96149dd0be5e --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractAssertionGroupSelect.tsx @@ -0,0 +1,54 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Assertion } from '../../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../../constants'; +import { DataContractCategoryType } from './types'; +import { DatasetAssertionsList } from '../../DatasetAssertionsList'; + +const Category = styled.div` + padding: 20px; + font-weight: bold; + font-size: 14px; + background-color: ${ANTD_GRAY[3]}; + border-radius: 4px; +`; + +const Hint = styled.span` + font-weight: normal; + font-size: 14px; + color: ${ANTD_GRAY[8]}; +`; + +type Props = { + category: DataContractCategoryType; + multiple?: boolean; + assertions: Assertion[]; + selectedUrns: string[]; + onSelect: (assertionUrn: string) => void; +}; + +/** + * Used for selecting the assertions that are part of a data contract + */ +export const DataContractAssertionGroupSelect = ({ + category, + assertions, + multiple = true, + selectedUrns, + onSelect, +}: Props) => { + return ( + <> + + {category} {!multiple ? `(Choose 1)` : ''} + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilder.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilder.tsx new file mode 100644 index 0000000000000..e3d85cb9db7cd --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilder.tsx @@ -0,0 +1,161 @@ +import React, { useState } from 'react'; +import { message, Button } from 'antd'; +import styled from 'styled-components'; +import lodash from 'lodash'; +import { DataContract, AssertionType, Assertion } from '../../../../../../../../types.generated'; +import { DataContractBuilderState, DataContractCategoryType, DEFAULT_BUILDER_STATE } from './types'; +import { buildUpsertDataContractMutationVariables } from './utils'; +import { useUpsertDataContractMutation } from '../../../../../../../../graphql/contract.generated'; +import { createAssertionGroups } from '../../utils'; +import { DataContractAssertionGroupSelect } from './DataContractAssertionGroupSelect'; +import { ANTD_GRAY } from '../../../../../constants'; +import { DATA_QUALITY_ASSERTION_TYPES } from '../utils'; +import { useGetDatasetAssertionsQuery } from '../../../../../../../../graphql/dataset.generated'; + +const AssertionsSection = styled.div` + border: 0.5px solid ${ANTD_GRAY[4]}; +`; + +const HeaderText = styled.div` + padding: 16px 20px; + color: ${ANTD_GRAY[7]}; + font-size: 16px; +`; + +const ActionContainer = styled.div` + display: flex; + justify-content: space-between; + margin-top: 16px; +`; + +const CancelButton = styled(Button)` + margin-left: 12px; +`; + +const SaveButton = styled(Button)` + margin-right: 20px; +`; + +type Props = { + entityUrn: string; + initialState?: DataContractBuilderState; + onSubmit?: (contract: DataContract) => void; + onCancel?: () => void; +}; + +/** + * This component is a modal used for constructing new Data Contracts + * + * In order to build a data contract, we simply list all dataset assertions and allow the user to choose. + */ +export const DataContractBuilder = ({ entityUrn, initialState, onSubmit, onCancel }: Props) => { + const isEdit = !!initialState; + const [builderState, setBuilderState] = useState(initialState || DEFAULT_BUILDER_STATE); + const [upsertDataContractMutation] = useUpsertDataContractMutation(); + + // note that for contracts, we do not allow the use of sibling node assertions, for clarity. + const { data } = useGetDatasetAssertionsQuery({ + variables: { urn: entityUrn }, + fetchPolicy: 'cache-first', + }); + const assertionData = data?.dataset?.assertions?.assertions ?? []; + + const assertionGroups = createAssertionGroups(assertionData as Array); + const freshnessAssertions = + assertionGroups.find((group) => group.type === AssertionType.Freshness)?.assertions || []; + const schemaAssertions = assertionGroups.find((group) => group.type === AssertionType.DataSchema)?.assertions || []; + const dataQualityAssertions = assertionGroups + .filter((group) => DATA_QUALITY_ASSERTION_TYPES.has(group.type)) + .flatMap((group) => group.assertions || []); + + /** + * Upserts the Data Contract for an entity + */ + const upsertDataContract = () => { + return upsertDataContractMutation({ + variables: buildUpsertDataContractMutationVariables(entityUrn, builderState), + }) + .then(({ data: dataContract, errors }) => { + if (!errors) { + message.success({ + content: isEdit ? `Edited Data Contract` : `Created Data Contract!`, + duration: 3, + }); + onSubmit?.(dataContract?.upsertDataContract as DataContract); + } + }) + .catch(() => { + message.destroy(); + message.error({ content: 'Failed to create Data Contract! An unexpected error occurred' }); + }); + }; + + const onSelectDataAssertion = (assertionUrn: string, type: string) => { + const selected = builderState[type]?.some((c) => c.assertionUrn === assertionUrn); + if (selected) { + setBuilderState({ + ...builderState, + [type]: builderState[type]?.filter((c) => c.assertionUrn !== assertionUrn), + }); + } else { + setBuilderState({ + ...builderState, + [type]: [...(builderState[type] || []), { assertionUrn }], + }); + } + }; + + const editDisabled = + lodash.isEqual(builderState, initialState) || lodash.isEqual(builderState, DEFAULT_BUILDER_STATE); + + const hasAssertions = freshnessAssertions.length || schemaAssertions.length || dataQualityAssertions.length; + + return ( + <> + {(hasAssertions && Select the assertions that will make up your contract.) || ( + Add a few assertions on this entity to create a data contract out of them. + )} + + {(freshnessAssertions.length && ( + onSelectDataAssertion(selectedUrn, 'freshness')} + /> + )) || + undefined} + {(schemaAssertions.length && ( + onSelectDataAssertion(selectedUrn, 'schema')} + /> + )) || + undefined} + {(dataQualityAssertions.length && ( + c.assertionUrn) || []} + onSelect={(selectedUrn: string) => onSelectDataAssertion(selectedUrn, 'dataQuality')} + /> + )) || + undefined} + + + Cancel +
+ + Save + +
+ + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilderModal.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilderModal.tsx new file mode 100644 index 0000000000000..75a8fe0410918 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/DataContractBuilderModal.tsx @@ -0,0 +1,67 @@ +import React from 'react'; +import { Modal, Typography } from 'antd'; +import { DataContract } from '../../../../../../../../types.generated'; +import ClickOutside from '../../../../../../../shared/ClickOutside'; +import { DataContractBuilderState } from './types'; +import { DataContractBuilder } from './DataContractBuilder'; + +const modalStyle = {}; +const modalBodyStyle = { + paddingRight: 0, + paddingLeft: 0, + paddingBottom: 20, + paddingTop: 0, + maxHeight: '70vh', + 'overflow-x': 'auto', +}; + +type Props = { + entityUrn: string; + initialState?: DataContractBuilderState; + onSubmit?: (contract: DataContract) => void; + onCancel?: () => void; +}; + +/** + * This component is a modal used for constructing new Data Contracts + */ +export const DataContractBuilderModal = ({ entityUrn, initialState, onSubmit, onCancel }: Props) => { + const isEditing = initialState !== undefined; + const titleText = isEditing ? 'Edit Data Contract' : 'New Data Contract'; + + const modalClosePopup = () => { + Modal.confirm({ + title: 'Exit Editor', + content: `Are you sure you want to exit the editor? All changes will be lost`, + onOk() { + onCancel?.(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }; + + return ( + + {titleText}} + style={modalStyle} + bodyStyle={modalBodyStyle} + visible + onCancel={onCancel} + > + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/types.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/types.ts new file mode 100644 index 0000000000000..497bd03cdf874 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/types.ts @@ -0,0 +1,38 @@ +/** + * The Data Contract Builder state + */ +export type DataContractBuilderState = { + /** + * The schema contract. In the UI, we only support defining a single schema contract. + */ + schema?: { + assertionUrn: string; + }; + + /** + * The freshness contract. In the UI, we only support defining a single freshness contract. + */ + freshness?: { + assertionUrn: string; + }; + + /** + * Data Quality contract. We cane define multiple data quality rules as part of the contract. + */ + dataQuality?: { + assertionUrn: string; + }[]; + +}; + +export const DEFAULT_BUILDER_STATE = { + dataQuality: undefined, + schema: undefined, + freshness: undefined, +}; + +export enum DataContractCategoryType { + FRESHNESS = 'Freshness', + SCHEMA = 'Schema', + DATA_QUALITY = 'Data Quality', +} diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts new file mode 100644 index 0000000000000..da2ae66d1ec9c --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/builder/utils.ts @@ -0,0 +1,120 @@ +import { DataContract } from '../../../../../../../../types.generated'; +import { DataContractBuilderState, DataContractCategoryType } from './types'; + +/** + * Creates a builder state instance from a Data Contract object. + */ +export const createBuilderState = (contract?: DataContract | null): DataContractBuilderState | undefined => { + if (contract) { + return { + schema: + (contract?.properties?.schema?.length && { + assertionUrn: contract?.properties?.schema[0].assertion.urn, + }) || + undefined, + freshness: + (contract?.properties?.freshness?.length && { + assertionUrn: contract?.properties?.freshness[0].assertion.urn, + }) || + undefined, + dataQuality: + contract?.properties?.dataQuality?.map((c) => ({ assertionUrn: c.assertion.urn })) || undefined, + }; + } + return undefined; +}; + +/** + * Constructs the input variables required for upserting a data contract using graphql + */ +export const buildUpsertDataContractMutationVariables = (entityUrn: string, state: DataContractBuilderState) => { + return { + input: { + entityUrn, + freshness: (state.freshness && [state.freshness]) || [], + schema: (state.schema && [state.schema]) || [], + dataQuality: state.dataQuality || [], + }, + }; +}; + +/** + * Constructs the input variables required for removing an assertion from a data contract using graphql. + */ +export const buildRemoveAssertionFromContractMutationVariables = ( + entityUrn: string, + assertionUrn: string, + contract?: DataContract, +) => { + return { + input: { + entityUrn, + freshness: contract?.properties?.freshness + ?.filter((c) => c.assertion.urn !== assertionUrn) + ?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + schema: contract?.properties?.schema + ?.filter((c) => c.assertion.urn !== assertionUrn) + ?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + dataQuality: contract?.properties?.dataQuality + ?.filter((c) => c.assertion.urn !== assertionUrn) + ?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + }, + }; +}; + +/** + * Constructs the input variables required for adding an assertion to a data contract using graphql. + */ +export const buildAddAssertionToContractMutationVariables = ( + category: DataContractCategoryType, + entityUrn: string, + assertionUrn: string, + contract?: DataContract, +) => { + const baseInput = { + entityUrn, + freshness: contract?.properties?.freshness?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + schema: contract?.properties?.schema?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + dataQuality: contract?.properties?.dataQuality?.map((c) => ({ + assertionUrn: c.assertion.urn, + })), + }; + + switch (category) { + case DataContractCategoryType.SCHEMA: + // Replace schema assertion. We only support 1 schema assertion at a time (currently). + return { + input: { + ...baseInput, + schema: [{ assertionUrn }], + }, + }; + case DataContractCategoryType.FRESHNESS: + // Replace freshness assertion. We only support 1 freshness assertion at a time (currently). + return { + input: { + ...baseInput, + freshness: [{ assertionUrn }], + }, + }; + case DataContractCategoryType.DATA_QUALITY: + return { + input: { + ...baseInput, + dataQuality: [...(baseInput.dataQuality || []), { assertionUrn }], + }, + }; + default: + throw new Error(`Unrecognized category type ${category} provided.`); + } +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/utils.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/utils.tsx new file mode 100644 index 0000000000000..cc2e1bb7b386e --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/contract/utils.tsx @@ -0,0 +1,114 @@ +import React from 'react'; +import { + CheckOutlined, + ClockCircleOutlined, + CloseOutlined, + ExclamationCircleFilled, + StopOutlined, +} from '@ant-design/icons'; +import { Assertion, AssertionType, DataContract, DataContractState } from '../../../../../../../types.generated'; +import { ANTD_GRAY } from '../../../../constants'; +import { FAILURE_COLOR_HEX, SUCCESS_COLOR_HEX, WARNING_COLOR_HEX } from '../utils'; +import { DataContractCategoryType } from './builder/types'; +import { AssertionStatusSummary } from '../types'; + +export const getContractSummaryIcon = (state: DataContractState, summary: AssertionStatusSummary) => { + if (state === DataContractState.Pending) { + return ; + } + if (summary.total === 0) { + return ; + } + if (summary.passing === summary.total) { + return ; + } + if (summary.failing > 0) { + return ; + } + if (summary.erroring > 0) { + return ; + } + return ; +}; + +export const getContractSummaryTitle = (state: DataContractState, summary: AssertionStatusSummary) => { + if (state === DataContractState.Pending) { + return 'This contract is pending implementation'; + } + if (summary.total === 0) { + return 'This contract has not yet been validated'; + } + if (summary.passing === summary.total) { + return 'This dataset is meeting its contract'; + } + if (summary.failing > 0) { + return 'This dataset is not meeting its contract'; + } + if (summary.erroring > 0) { + return 'Unable to determine contract status'; + } + return 'This contract has not yet been validated'; +}; + +export const getContractSummaryMessage = (state: DataContractState, summary: AssertionStatusSummary) => { + if (state === DataContractState.Pending) { + return 'This may take some time. Come back later!'; + } + if (summary.total === 0) { + return 'No contract assertions have been run yet'; + } + if (summary.passing === summary.total) { + return 'All contract assertions are passing'; + } + if (summary.failing > 0) { + return 'Some contract assertions are failing'; + } + if (summary.erroring > 0) { + return 'Some contract assertions are completing with errors'; + } + return 'No contract assertions have been run yet'; +}; + +/** + * Returns true if a given assertion is part of a given contract, false otherwise. + */ +export const isAssertionPartOfContract = (assertion: Assertion, contract: DataContract) => { + if (contract.properties?.dataQuality?.some((c) => c.assertion.urn === assertion.urn)) { + return true; + } + if (contract.properties?.schema?.some((c) => c.assertion.urn === assertion.urn)) { + return true; + } + if (contract.properties?.freshness?.some((c) => c.assertion.urn === assertion.urn)) { + return true; + } + return false; +}; + +/** + * Retrieves the high level contract category - schema, freshness, or data quality - given an assertion + */ +export const getDataContractCategoryFromAssertion = (assertion: Assertion) => { + if ( + assertion.info?.type === AssertionType.Dataset || + assertion.info?.type === AssertionType.Volume || + assertion.info?.type === AssertionType.Field || + assertion.info?.type === AssertionType.Sql + ) { + return DataContractCategoryType.DATA_QUALITY; + } + if (assertion.info?.type === AssertionType.Freshness) { + return DataContractCategoryType.FRESHNESS; + } + if (assertion.info?.type === AssertionType.DataSchema) { + return DataContractCategoryType.SCHEMA; + } + return DataContractCategoryType.DATA_QUALITY; +}; + +export const DATA_QUALITY_ASSERTION_TYPES = new Set([ + AssertionType.Volume, + AssertionType.Sql, + AssertionType.Field, + AssertionType.Dataset, +]); diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/fieldDescriptionUtils.ts b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/fieldDescriptionUtils.ts new file mode 100644 index 0000000000000..3c6e14f1d80ab --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/fieldDescriptionUtils.ts @@ -0,0 +1,179 @@ +import { + AssertionStdOperator, + AssertionStdParameters, + FieldAssertionInfo, + FieldAssertionType, + FieldMetricType, + FieldTransformType, +} from '../../../../../../types.generated'; +import { formatNumberWithoutAbbreviation } from '../../../../../shared/formatNumber'; +import { parseMaybeStringAsFloatOrDefault } from '../../../../../shared/numberUtil'; + +const ASSERTION_OPERATOR_TO_DESCRIPTION: Record = { + [AssertionStdOperator.EqualTo]: 'Is equal to', + [AssertionStdOperator.NotEqualTo]: 'Is not equal to', + [AssertionStdOperator.Contain]: 'Contains', + [AssertionStdOperator.RegexMatch]: 'Matches', + [AssertionStdOperator.StartWith]: 'Starts with', + [AssertionStdOperator.EndWith]: 'Ends with', + [AssertionStdOperator.In]: 'Is in', + [AssertionStdOperator.NotIn]: 'Is not in', + + [AssertionStdOperator.IsFalse]: 'Is False', + [AssertionStdOperator.IsTrue]: 'Is True', + [AssertionStdOperator.Null]: 'Is NULL', + [AssertionStdOperator.NotNull]: 'Is not NULL', + + [AssertionStdOperator.GreaterThan]: 'Is greater than', + [AssertionStdOperator.GreaterThanOrEqualTo]: 'Is greater than or equal to', + [AssertionStdOperator.LessThan]: 'Is less than', + [AssertionStdOperator.LessThanOrEqualTo]: 'Is less than or equal to', + [AssertionStdOperator.Between]: 'Is within a range', + + [AssertionStdOperator.Native]: undefined, +}; + +const SUPPORTED_OPERATORS_FOR_FIELD_DESCRIPTION = [ + AssertionStdOperator.EqualTo, + AssertionStdOperator.Null, + AssertionStdOperator.NotNull, + AssertionStdOperator.NotEqualTo, + AssertionStdOperator.NotIn, + AssertionStdOperator.RegexMatch, + AssertionStdOperator.GreaterThan, + AssertionStdOperator.LessThan, + AssertionStdOperator.GreaterThanOrEqualTo, + AssertionStdOperator.LessThanOrEqualTo, + AssertionStdOperator.In, + AssertionStdOperator.Between, + AssertionStdOperator.Contain, + AssertionStdOperator.IsTrue, + AssertionStdOperator.IsFalse, +]; + +const getAssertionStdOperator = (operator: AssertionStdOperator) => { + if (!ASSERTION_OPERATOR_TO_DESCRIPTION[operator] || !SUPPORTED_OPERATORS_FOR_FIELD_DESCRIPTION.includes(operator)) { + throw new Error(`Unknown operator ${operator}`); + } + return ASSERTION_OPERATOR_TO_DESCRIPTION[operator]?.toLowerCase(); +}; + +export const getFieldMetricTypeReadableLabel = (metric: FieldMetricType) => { + switch (metric) { + case FieldMetricType.NullCount: + return 'Null count'; + case FieldMetricType.NullPercentage: + return 'Null percentage'; + case FieldMetricType.UniqueCount: + return 'Unique count'; + case FieldMetricType.UniquePercentage: + return 'Unique percentage'; + case FieldMetricType.MaxLength: + return 'Max length'; + case FieldMetricType.MinLength: + return 'Min length'; + case FieldMetricType.EmptyCount: + return 'Empty count'; + case FieldMetricType.EmptyPercentage: + return 'Empty percentage'; + case FieldMetricType.Max: + return 'Max'; + case FieldMetricType.Min: + return 'Min'; + case FieldMetricType.Mean: + return 'Average'; + case FieldMetricType.Median: + return 'Median'; + case FieldMetricType.NegativeCount: + return 'Negative count'; + case FieldMetricType.NegativePercentage: + return 'Negative percentage'; + case FieldMetricType.Stddev: + return 'Standard deviation'; + case FieldMetricType.ZeroCount: + return 'Zero count'; + case FieldMetricType.ZeroPercentage: + return 'Zero percentage'; + default: + throw new Error(`Unknown field metric type ${metric}`); + } +}; + +const getFieldTransformType = (transform: FieldTransformType) => { + switch (transform) { + case FieldTransformType.Length: + return 'Length'; + default: + throw new Error(`Unknown field transform type ${transform}`); + } +}; + +const getAssertionStdParameters = (parameters: AssertionStdParameters) => { + if (parameters.value) { + return formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.value.value, parameters.value.value), + ); + } + if (parameters.minValue && parameters.maxValue) { + return `${formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.minValue.value, parameters.minValue.value), + )} and ${formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.maxValue.value, parameters.maxValue.value), + )}`; + } + return ''; +}; + +export const getFieldDescription = (assertionInfo: FieldAssertionInfo) => { + const { type, fieldValuesAssertion, fieldMetricAssertion } = assertionInfo; + switch (type) { + case FieldAssertionType.FieldValues: + return fieldValuesAssertion?.field?.path; + case FieldAssertionType.FieldMetric: + return fieldMetricAssertion?.field?.path; + default: + throw new Error(`Unknown field assertion type ${type}`); + } +}; + +export const getFieldOperatorDescription = (assertionInfo: FieldAssertionInfo) => { + const { type, fieldValuesAssertion, fieldMetricAssertion } = assertionInfo; + switch (type) { + case FieldAssertionType.FieldValues: + if (!fieldValuesAssertion?.operator) return ''; + return getAssertionStdOperator(fieldValuesAssertion.operator); + case FieldAssertionType.FieldMetric: + if (!fieldMetricAssertion?.operator) return ''; + return getAssertionStdOperator(fieldMetricAssertion.operator); + default: + throw new Error(`Unknown field assertion type ${type}`); + } +}; + +export const getFieldTransformDescription = (assertionInfo: FieldAssertionInfo) => { + const { type, fieldValuesAssertion, fieldMetricAssertion } = assertionInfo; + switch (type) { + case FieldAssertionType.FieldValues: + if (!fieldValuesAssertion?.transform?.type) return ''; + return getFieldTransformType(fieldValuesAssertion.transform.type); + case FieldAssertionType.FieldMetric: + if (!fieldMetricAssertion?.metric) return ''; + return getFieldMetricTypeReadableLabel(fieldMetricAssertion.metric); + default: + throw new Error(`Unknown field assertion type ${type}`); + } +}; + +export const getFieldParametersDescription = (assertionInfo: FieldAssertionInfo) => { + const { type, fieldValuesAssertion, fieldMetricAssertion } = assertionInfo; + switch (type) { + case FieldAssertionType.FieldValues: + if (!fieldValuesAssertion?.parameters) return ''; + return getAssertionStdParameters(fieldValuesAssertion.parameters); + case FieldAssertionType.FieldMetric: + if (!fieldMetricAssertion?.parameters) return ''; + return getAssertionStdParameters(fieldMetricAssertion.parameters); + default: + throw new Error(`Unknown field assertion type ${type}`); + } +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/shared/styledComponents.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/shared/styledComponents.tsx new file mode 100644 index 0000000000000..14651899a355f --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/shared/styledComponents.tsx @@ -0,0 +1,32 @@ +import styled from 'styled-components'; +import { CheckOutlined, ClockCircleOutlined, CloseOutlined, ExclamationCircleOutlined } from '@ant-design/icons'; +import { ANTD_GRAY } from '../../../../constants'; +import { FAILURE_COLOR_HEX, SUCCESS_COLOR_HEX, WARNING_COLOR_HEX } from '../utils'; + +export const StyledCheckOutlined = styled(CheckOutlined)` + color: ${SUCCESS_COLOR_HEX}; + font-size: 16px; + margin-right: 4px; + margin-left: 4px; +`; + +export const StyledCloseOutlined = styled(CloseOutlined)` + color: ${FAILURE_COLOR_HEX}; + font-size: 16px; + margin-right: 4px; + margin-left: 4px; +`; + +export const StyledExclamationOutlined = styled(ExclamationCircleOutlined)` + color: ${WARNING_COLOR_HEX}; + font-size: 16px; + margin-right: 4px; + margin-left: 4px; +`; + +export const StyledClockCircleOutlined = styled(ClockCircleOutlined)` + color: ${ANTD_GRAY[6]}; + font-size: 16px; + margin-right: 4px; + margin-left: 4px; +`; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/types.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/types.tsx new file mode 100644 index 0000000000000..8a70a3d87c147 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/types.tsx @@ -0,0 +1,21 @@ +import { Assertion, AssertionType } from '../../../../../../types.generated'; + +export type AssertionStatusSummary = { + passing: number; + failing: number; + erroring: number; + total: number; // Total assertions with at least 1 run. + totalAssertions: number; +}; + +/** + * A group of assertions related by their logical type or category. + */ +export type AssertionGroup = { + name: string; + icon: React.ReactNode; + description?: string; + assertions: Assertion[]; + summary: AssertionStatusSummary; + type: AssertionType; +}; diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/utils.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/utils.tsx new file mode 100644 index 0000000000000..e01f2ad2eae83 --- /dev/null +++ b/datahub-web-react/src/app/entity/shared/tabs/Dataset/Validations/utils.tsx @@ -0,0 +1,378 @@ +import React from 'react'; +import styled from 'styled-components'; +import { + ClockCircleOutlined, + TableOutlined, + ProjectOutlined, + ConsoleSqlOutlined, + CheckOutlined, + CloseOutlined, + ApiOutlined, + CodeOutlined, + ExclamationCircleOutlined, +} from '@ant-design/icons'; +import { + Assertion, + AssertionResultType, + AssertionStdOperator, + AssertionStdParameters, + AssertionType, + AssertionValueChangeType, + EntityType, + VolumeAssertionInfo, + VolumeAssertionType, +} from '../../../../../../types.generated'; +import { sortAssertions } from './assertionUtils'; +import { AssertionGroup, AssertionStatusSummary } from './types'; +import { formatNumberWithoutAbbreviation } from '../../../../../shared/formatNumber'; +import { parseMaybeStringAsFloatOrDefault } from '../../../../../shared/numberUtil'; + +export const SUCCESS_COLOR_HEX = '#52C41A'; +export const FAILURE_COLOR_HEX = '#F5222D'; +export const WARNING_COLOR_HEX = '#FA8C16'; + +const StyledClockCircleOutlined = styled(ClockCircleOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 14px; + } +`; + +const StyledTableOutlined = styled(TableOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +const StyledProjectOutlined = styled(ProjectOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +const StyledConsoleSqlOutlined = styled(ConsoleSqlOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +const StyledApiOutlined = styled(ApiOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +const StyledCheckOutlined = styled(CheckOutlined)` + && { + color: ${SUCCESS_COLOR_HEX}; + font-size: 14px; + padding: 0px; + margin: 0px; + } +`; + +const StyledCloseOutlined = styled(CloseOutlined)` + && { + color: ${FAILURE_COLOR_HEX}; + font-size: 14px; + padding: 0px; + margin: 0px; + } +`; + +const StyledExclamationOutlined = styled(ExclamationCircleOutlined)` + && { + color: ${WARNING_COLOR_HEX}; + font-size: 14px; + padding: 0px; + margin: 0px; + } +`; + +const StyledCodeOutlined = styled(CodeOutlined)` + && { + margin: 0px; + padding: 0px; + margin-right: 8px; + font-size: 18px; + } +`; + +export const ASSERTION_INFO = [ + { + name: 'Freshness', + description: 'Define & monitor your expectations about when this dataset should be updated', + icon: , + type: AssertionType.Freshness, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + }, + { + name: 'Volume', + description: 'Define & monitor your expectations about the size of this dataset', + icon: , + type: AssertionType.Volume, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + }, + { + name: 'Column', + description: 'Define & monitor your expectations about the values in a column', + icon: , + type: AssertionType.Field, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + requiresConnectionSupportedByMonitors: false, + }, + { + name: 'Schema', + description: "Define & monitor your expectations about the table's columns and their types", + icon: , + type: AssertionType.DataSchema, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + }, + { + name: 'Custom', + description: 'Define & monitor your expectations using custom SQL rules', + icon: , + type: AssertionType.Sql, + entityTypes: [EntityType.Dataset], + enabled: true, + visible: true, + requiresConnectionSupportedByMonitors: true, + }, + { + name: 'Other', + description: 'Assertions that are defined and maintained outside of DataHub.', + icon: , + type: AssertionType.Dataset, + entityTypes: [EntityType.Dataset], + enabled: false, + visible: false, + }, +]; + +const ASSERTION_TYPE_TO_INFO = new Map(); +ASSERTION_INFO.forEach((info) => { + ASSERTION_TYPE_TO_INFO.set(info.type, info); +}); + +const getAssertionGroupName = (type: AssertionType): string => { + return ASSERTION_TYPE_TO_INFO.has(type) ? ASSERTION_TYPE_TO_INFO.get(type).name : 'Unknown'; +}; + +const getAssertionGroupTypeIcon = (type: AssertionType) => { + return ASSERTION_TYPE_TO_INFO.has(type) ? ASSERTION_TYPE_TO_INFO.get(type).icon : undefined; +}; + +/** + * Returns a status summary for the assertions associated with a Dataset. + * + * @param assertions The assertions to extract the summary for + */ +export const getAssertionsSummary = (assertions: Assertion[]): AssertionStatusSummary => { + const summary = { + passing: 0, + failing: 0, + erroring: 0, + total: 0, + totalAssertions: assertions.length, + }; + assertions.forEach((assertion) => { + if ((assertion.runEvents?.runEvents?.length || 0) > 0) { + const mostRecentRun = assertion.runEvents?.runEvents?.[0]; + const resultType = mostRecentRun?.result?.type; + if (AssertionResultType.Success === resultType) { + summary.passing++; + } + if (AssertionResultType.Failure === resultType) { + summary.failing++; + } + if (AssertionResultType.Error === resultType) { + summary.erroring++; + } + if (AssertionResultType.Init !== resultType) { + summary.total++; // only count assertions for which there is one completed run event, ignoring INIT statuses! + } + } + }); + return summary; +}; + +// /** +// * Returns a list of assertion groups, where assertions are grouped +// * by their "type" or "category". Each group includes the assertions inside, along with +// * a summary of passing and failing assertions for the group. +// * +// * @param assertions The assertions to group +// */ +export const createAssertionGroups = (assertions: Array): AssertionGroup[] => { + // Pre-sort the list of assertions based on which has been most recently executed. + assertions.sort(sortAssertions); + + const typeToAssertions = new Map(); + assertions + .filter((assertion) => assertion.info?.type) + .forEach((assertion) => { + const groupType = assertion.info?.type; + const groupedAssertions = typeToAssertions.get(groupType) || []; + groupedAssertions.push(assertion); + typeToAssertions.set(groupType, groupedAssertions); + }); + + // Now, create summary for each type and build the AssertionGroup object + const assertionGroups: AssertionGroup[] = []; + typeToAssertions.forEach((groupedAssertions, type) => { + const newGroup: AssertionGroup = { + name: getAssertionGroupName(type), + icon: getAssertionGroupTypeIcon(type), + assertions: groupedAssertions, + summary: getAssertionsSummary(groupedAssertions), + type, + }; + assertionGroups.push(newGroup); + }); + + return assertionGroups; +}; + +export const getAssertionGroupSummaryIcon = (summary: AssertionStatusSummary) => { + if (summary.total === 0) { + return null; + } + if (summary.passing === summary.total) { + return ; + } + if (summary.erroring > 0) { + return ; + } + return ; +}; + +export const getAssertionGroupSummaryMessage = (summary: AssertionStatusSummary) => { + if (summary.total === 0) { + return 'No assertions have run'; + } + if (summary.passing === summary.total) { + return 'All assertions are passing'; + } + if (summary.erroring > 0) { + return 'An error is preventing some assertions from running'; + } + if (summary.failing === summary.total) { + return 'All assertions are failing'; + } + return 'Some assertions are failing'; +}; + +export const getAssertionTypesForEntityType = (entityType: EntityType, monitorsConnectionForEntityExists: boolean) => { + return ASSERTION_INFO.filter((type) => type.entityTypes.includes(entityType)).map((type) => ({ + ...type, + enabled: type.enabled && (!type.requiresConnectionSupportedByMonitors || monitorsConnectionForEntityExists), + })); +}; + +type VolumeTypeField = + | 'rowCountTotal' + | 'rowCountChange' + | 'incrementingSegmentRowCountTotal' + | 'incrementingSegmentRowCountChange'; + +export const getPropertyFromVolumeType = (type: VolumeAssertionType) => { + switch (type) { + case VolumeAssertionType.RowCountTotal: + return 'rowCountTotal' as VolumeTypeField; + case VolumeAssertionType.RowCountChange: + return 'rowCountChange' as VolumeTypeField; + case VolumeAssertionType.IncrementingSegmentRowCountTotal: + return 'incrementingSegmentRowCountTotal' as VolumeTypeField; + case VolumeAssertionType.IncrementingSegmentRowCountChange: + return 'incrementingSegmentRowCountChange' as VolumeTypeField; + default: + throw new Error(`Unknown volume assertion type: ${type}`); + } +}; + +export const getVolumeTypeInfo = (volumeAssertion: VolumeAssertionInfo) => { + const result = volumeAssertion[getPropertyFromVolumeType(volumeAssertion.type)]; + if (!result) { + return undefined; + } + return result; +}; + +export const getIsRowCountChange = (type: VolumeAssertionType) => { + return [VolumeAssertionType.RowCountChange, VolumeAssertionType.IncrementingSegmentRowCountChange].includes(type); +}; + +export const getVolumeTypeDescription = (volumeType: VolumeAssertionType) => { + switch (volumeType) { + case VolumeAssertionType.RowCountTotal: + case VolumeAssertionType.IncrementingSegmentRowCountTotal: + return 'has'; + case VolumeAssertionType.RowCountChange: + case VolumeAssertionType.IncrementingSegmentRowCountChange: + return 'should grow by'; + default: + throw new Error(`Unknown volume type ${volumeType}`); + } +}; + +export const getOperatorDescription = (operator: AssertionStdOperator) => { + switch (operator) { + case AssertionStdOperator.GreaterThanOrEqualTo: + return 'at least'; + case AssertionStdOperator.LessThanOrEqualTo: + return 'at most'; + case AssertionStdOperator.Between: + return 'between'; + default: + throw new Error(`Unknown operator ${operator}`); + } +}; + +export const getValueChangeTypeDescription = (valueChangeType: AssertionValueChangeType) => { + switch (valueChangeType) { + case AssertionValueChangeType.Absolute: + return 'rows'; + case AssertionValueChangeType.Percentage: + return '%'; + default: + throw new Error(`Unknown value change type ${valueChangeType}`); + } +}; + +export const getParameterDescription = (parameters: AssertionStdParameters) => { + if (parameters.value) { + return formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.value.value, parameters.value.value), + ); + } + if (parameters.minValue && parameters.maxValue) { + return `${formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.minValue.value, parameters.minValue.value), + )} and ${formatNumberWithoutAbbreviation( + parseMaybeStringAsFloatOrDefault(parameters.maxValue.value, parameters.maxValue.value), + )}`; + } + throw new Error('Invalid assertion parameters provided'); +}; diff --git a/datahub-web-react/src/app/shared/numberUtil.ts b/datahub-web-react/src/app/shared/numberUtil.ts new file mode 100644 index 0000000000000..6b6b76b04d0d1 --- /dev/null +++ b/datahub-web-react/src/app/shared/numberUtil.ts @@ -0,0 +1,24 @@ + +export function parseMaybeStringAsFloatOrDefault(str: any, fallback?: T): number | T | undefined { + const parsedValue = typeof str === 'string' ? parseFloat(str) : str; + return typeof parsedValue === 'number' && !Number.isNaN(parsedValue) ? parsedValue : fallback; +} + + +export function parseJsonArrayOrDefault(str: any, fallback: T[] = []): T[] | undefined { + // Check if the input is a string and try to parse it. + if (typeof str === 'string') { + try { + const parsedValue = JSON.parse(str); + // Check if the parsed value is an array before returning. + if (Array.isArray(parsedValue)) { + return parsedValue; + } + } catch (e) { + // If parsing throws, log the error (optional) and proceed to return fallback. + console.error("Failed to parse JSON:", e); + } + } + // Return fallback if the above conditions fail. + return fallback; +} \ No newline at end of file diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index b4f16e2d2a824..4262f772b006e 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -53,6 +53,7 @@ export const DEFAULT_APP_CONFIG = { nestedDomainsEnabled: true, platformBrowseV2: false, businessAttributeEntityEnabled: false, + dataContractsEnabled: false, }, }; diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index 7b47fc0302247..662b2f11336fa 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -68,6 +68,7 @@ query appConfig { nestedDomainsEnabled platformBrowseV2 businessAttributeEntityEnabled + dataContractsEnabled } } } diff --git a/datahub-web-react/src/graphql/assertion.graphql b/datahub-web-react/src/graphql/assertion.graphql index 8cf61038e4dab..15f1d6484fc0b 100644 --- a/datahub-web-react/src/graphql/assertion.graphql +++ b/datahub-web-react/src/graphql/assertion.graphql @@ -95,3 +95,15 @@ query getAssertionRuns($assertionUrn: String!, $startTime: Long, $endTime: Long, mutation deleteAssertion($urn: String!) { deleteAssertion(urn: $urn) } + +fragment assertionDetailsWithRunEvents on Assertion { + ...assertionDetails + runEvents(status: COMPLETE, limit: 1) { + total + failed + succeeded + runEvents { + ...assertionRunEventDetails + } + } +} diff --git a/datahub-web-react/src/graphql/contract.graphql b/datahub-web-react/src/graphql/contract.graphql new file mode 100644 index 0000000000000..4c54dba037a35 --- /dev/null +++ b/datahub-web-react/src/graphql/contract.graphql @@ -0,0 +1,53 @@ +fragment contractAssertionParameters on AssertionStdParameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } +} + +fragment dataContractDetails on DataContract { + urn + type + properties { + freshness { + assertion { + ...assertionDetailsWithRunEvents + } + } + schema { + assertion { + ...assertionDetailsWithRunEvents + } + } + dataQuality { + assertion { + ...assertionDetailsWithRunEvents + } + } + } + status { + state + } +} + +query getDatasetContract($urn: String!) { + dataset(urn: $urn) { + contract { + ...dataContractDetails + } + } +} + +mutation upsertDataContract($input: UpsertDataContractInput!) { + upsertDataContract(input: $input) { + ...dataContractDetails + } +} diff --git a/metadata-service/configuration/src/main/resources/application.yaml b/metadata-service/configuration/src/main/resources/application.yaml index 9125bb046d7c8..14de0530f51f0 100644 --- a/metadata-service/configuration/src/main/resources/application.yaml +++ b/metadata-service/configuration/src/main/resources/application.yaml @@ -370,6 +370,7 @@ featureFlags: nestedDomainsEnabled: ${NESTED_DOMAINS_ENABLED:true} # Enables the nested Domains feature that allows users to have sub-Domains. If this is off, Domains appear "flat" again schemaFieldEntityFetchEnabled: ${SCHEMA_FIELD_ENTITY_FETCH_ENABLED:true} # Enables fetching for schema field entities from the database when we hydrate them on schema fields businessAttributeEntityEnabled: ${BUSINESS_ATTRIBUTE_ENTITY_ENABLED:false} # Enables business attribute entity which can be associated with field of dataset + dataContractsEnabled: ${DATA_CONTRACTS_ENABLED:false} # Enables the Data Contracts feature (Tab) in the UI entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true} @@ -476,7 +477,7 @@ metadataChangeProposal: maxAttempts: ${MCP_VERSIONED_MAX_ATTEMPTS:1000} initialIntervalMs: ${MCP_VERSIONED_INITIAL_INTERVAL_MS:100} multiplier: ${MCP_VERSIONED_MULTIPLIER:10} - maxIntervalMs: ${MCP_VERSIONED_MAX_INTERVAL_MS:30000} + maxIntervalMs: ${MCP_VERSIONED_MAX_INTERVAL_MS:30000} # Timeseries MCL topic timeseries: # Whether to throttle MCP processing based on MCL backlog