feat(ingestion-ui) Display ingestion sources in UI more dynamically (#5789)

Author: chriscollins3456

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -49,6 +49,7 @@
 import com.linkedin.datahub.graphql.generated.GlossaryNode;
 import com.linkedin.datahub.graphql.generated.GlossaryTerm;
 import com.linkedin.datahub.graphql.generated.GlossaryTermAssociation;
+import com.linkedin.datahub.graphql.generated.IngestionSource;
 import com.linkedin.datahub.graphql.generated.InstitutionalMemoryMetadata;
 import com.linkedin.datahub.graphql.generated.LineageRelationship;
 import com.linkedin.datahub.graphql.generated.ListAccessTokenResult;
@@ -1475,6 +1476,13 @@ private <T, K> DataLoader<K, DataFetcherResult<T>> createDataLoader(final Loadab
     }
 
     private void configureIngestionSourceResolvers(final RuntimeWiring.Builder builder) {
-        builder.type("IngestionSource", typeWiring -> typeWiring.dataFetcher("executions", new IngestionSourceExecutionRequestsResolver(entityClient)));
+        builder.type("IngestionSource", typeWiring -> typeWiring
+            .dataFetcher("executions", new IngestionSourceExecutionRequestsResolver(entityClient))
+            .dataFetcher("platform", new LoadableTypeResolver<>(dataPlatformType,
+                (env) -> {
+                    final IngestionSource ingestionSource = env.getSource();
+                    return ingestionSource.getPlatform() != null ? ingestionSource.getPlatform().getUrn() : null;
+                })
+            ));
     }
 }

datahub-graphql-core/src/main/resources/ingestion.graphql

@@ -389,6 +389,11 @@ type IngestionSource {
   """
   schedule: IngestionSchedule
 
+  """
+  The data platform associated with this ingestion source
+  """
+  platform: DataPlatform
+
   """
   An type-specific set of configurations for the ingestion source
   """

datahub-web-react/src/app/ingest/source/IngestionSourceTable.tsx

@@ -108,6 +108,7 @@ function IngestionSourceTable({
         urn: source.urn,
         type: source.type,
         name: source.name,
+        platformUrn: source.platform?.urn,
         schedule: source.schedule?.interval,
         timezone: source.schedule?.timezone,
         execCount: source.executions?.total || 0,

datahub-web-react/src/app/ingest/source/IngestionSourceTableColumns.tsx

@@ -6,12 +6,12 @@ import React from 'react';
 import styled from 'styled-components/macro';
 import { ANTD_GRAY } from '../../entity/shared/constants';
 import { capitalizeFirstLetter } from '../../shared/textUtil';
+import useGetSourceLogoUrl from './builder/useGetSourceLogoUrl';
 import {
     getExecutionRequestStatusDisplayColor,
     getExecutionRequestStatusDisplayText,
     getExecutionRequestStatusIcon,
     RUNNING,
-    sourceTypeToIconUrl,
 } from './utils';
 
 const PreviewImage = styled(Image)`
@@ -63,7 +63,7 @@ const CliBadge = styled.span`
 `;
 
 export function TypeColumn(type: string, record: any) {
-    const iconUrl = sourceTypeToIconUrl(type);
+    const iconUrl = useGetSourceLogoUrl(type);
     const typeDisplayName = capitalizeFirstLetter(type);
 
     return (

datahub-web-react/src/app/ingest/source/builder/DefineRecipeStep.tsx

@@ -2,7 +2,7 @@ import { Alert, Button, Space, Typography } from 'antd';
 import React, { useEffect, useState } from 'react';
 import styled from 'styled-components';
 import { StepProps } from './types';
-import { getSourceConfigs, jsonToYaml } from '../utils';
+import { getPlaceholderRecipe, getSourceConfigs, jsonToYaml } from '../utils';
 import { YamlEditor } from './YamlEditor';
 import { ANTD_GRAY } from '../../../entity/shared/constants';
 import { IngestionSourceBuilderStep } from './steps';
@@ -37,13 +37,14 @@ const ControlsContainer = styled.div`
 /**
  * The step for defining a recipe
  */
-export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps) => {
+export const DefineRecipeStep = ({ state, updateState, goTo, prev, ingestionSources }: StepProps) => {
     const existingRecipeJson = state.config?.recipe;
     const existingRecipeYaml = existingRecipeJson && jsonToYaml(existingRecipeJson);
     const { type } = state;
-    const sourceConfigs = getSourceConfigs(type as string);
+    const sourceConfigs = getSourceConfigs(ingestionSources, type as string);
+    const placeholderRecipe = getPlaceholderRecipe(ingestionSources, type);
 
-    const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || sourceConfigs.placeholderRecipe);
+    const [stagedRecipeYml, setStagedRecipeYml] = useState(existingRecipeYaml || placeholderRecipe);
 
     useEffect(() => {
         if (existingRecipeYaml) {
@@ -54,12 +55,12 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps)
     const [stepComplete, setStepComplete] = useState(false);
 
     const isEditing: boolean = prev === undefined;
-    const displayRecipe = stagedRecipeYml || sourceConfigs.placeholderRecipe;
-    const sourceDisplayName = sourceConfigs.displayName;
-    const sourceDocumentationUrl = sourceConfigs.docsUrl; // Maybe undefined (in case of "custom")
+    const displayRecipe = stagedRecipeYml || placeholderRecipe;
+    const sourceDisplayName = sourceConfigs?.displayName;
+    const sourceDocumentationUrl = sourceConfigs?.docsUrl;
 
     // TODO: Delete LookML banner specific code
-    const isSourceLooker: boolean = sourceConfigs.type === 'looker';
+    const isSourceLooker: boolean = sourceConfigs?.name === 'looker';
     const [showLookerBanner, setShowLookerBanner] = useState(isSourceLooker && !isEditing);
 
     useEffect(() => {
@@ -90,6 +91,7 @@ export const DefineRecipeStep = ({ state, updateState, goTo, prev }: StepProps)
                 type={type}
                 isEditing={isEditing}
                 displayRecipe={displayRecipe}
+                sourceConfigs={sourceConfigs}
                 setStagedRecipe={setStagedRecipeYml}
                 onClickNext={onClickNext}
                 goToPrevious={prev}

datahub-web-react/src/app/ingest/source/builder/IngestionSourceBuilderModal.tsx

@@ -8,6 +8,7 @@ import { CreateScheduleStep } from './CreateScheduleStep';
 import { DefineRecipeStep } from './DefineRecipeStep';
 import { NameSourceStep } from './NameSourceStep';
 import { SelectTemplateStep } from './SelectTemplateStep';
+import sourcesJson from './sources.json';
 
 const ExpandButton = styled(Button)`
     && {
@@ -74,6 +75,8 @@ export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, o
     const [modalExpanded, setModalExpanded] = useState(false);
     const [ingestionBuilderState, setIngestionBuilderState] = useState<SourceBuilderState>({});
 
+    const ingestionSources = JSON.parse(JSON.stringify(sourcesJson)); // TODO: replace with call to server once we have access to dynamic list of sources
+
     // Reset the ingestion builder modal state when the modal is re-opened.
     const prevInitialState = useRef(initialState);
     useEffect(() => {
@@ -148,6 +151,7 @@ export const IngestionSourceBuilderModal = ({ initialState, visible, onSubmit, o
                 prev={stepStack.length > 1 ? prev : undefined}
                 submit={submit}
                 cancel={cancel}
+                ingestionSources={ingestionSources}
             />
         </Modal>
     );

datahub-web-react/src/app/ingest/source/builder/RecipeBuilder.tsx

@@ -6,6 +6,7 @@ import styled from 'styled-components/macro';
 import { ANTD_GRAY } from '../../../entity/shared/constants';
 import { YamlEditor } from './YamlEditor';
 import RecipeForm from './RecipeForm/RecipeForm';
+import { SourceConfig } from './types';
 
 export const ControlsContainer = styled.div`
     display: flex;
@@ -41,13 +42,14 @@ interface Props {
     type: string;
     isEditing: boolean;
     displayRecipe: string;
+    sourceConfigs?: SourceConfig;
     setStagedRecipe: (recipe: string) => void;
     onClickNext: () => void;
     goToPrevious?: () => void;
 }
 
 function RecipeBuilder(props: Props) {
-    const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props;
+    const { type, isEditing, displayRecipe, sourceConfigs, setStagedRecipe, onClickNext, goToPrevious } = props;
 
     const [isViewingForm, setIsViewingForm] = useState(true);
 
@@ -78,6 +80,7 @@ function RecipeBuilder(props: Props) {
                     type={type}
                     isEditing={isEditing}
                     displayRecipe={displayRecipe}
+                    sourceConfigs={sourceConfigs}
                     setStagedRecipe={setStagedRecipe}
                     onClickNext={onClickNext}
                     goToPrevious={goToPrevious}

datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx

@@ -10,6 +10,7 @@ import FormField from './FormField';
 import TestConnectionButton from './TestConnection/TestConnectionButton';
 import { useListSecretsQuery } from '../../../../../graphql/ingestion.generated';
 import { RecipeField, setFieldValueOnRecipe } from './common';
+import { SourceConfig } from '../types';
 
 export const ControlsContainer = styled.div`
     display: flex;
@@ -91,13 +92,14 @@ interface Props {
     type: string;
     isEditing: boolean;
     displayRecipe: string;
+    sourceConfigs?: SourceConfig;
     setStagedRecipe: (recipe: string) => void;
     onClickNext: () => void;
     goToPrevious?: () => void;
 }
 
 function RecipeForm(props: Props) {
-    const { type, isEditing, displayRecipe, setStagedRecipe, onClickNext, goToPrevious } = props;
+    const { type, isEditing, displayRecipe, sourceConfigs, setStagedRecipe, onClickNext, goToPrevious } = props;
     const { fields, advancedFields, filterFields, filterSectionTooltip } = RECIPE_FIELDS[type];
     const allFields = [...fields, ...advancedFields, ...filterFields];
     const { data, refetch: refetchSecrets } = useListSecretsQuery({
@@ -146,7 +148,7 @@ function RecipeForm(props: Props) {
                     ))}
                     {CONNECTORS_WITH_TEST_CONNECTION.has(type) && (
                         <TestConnectionWrapper>
-                            <TestConnectionButton type={type} recipe={displayRecipe} />
+                            <TestConnectionButton recipe={displayRecipe} sourceConfigs={sourceConfigs} />
                         </TestConnectionWrapper>
                     )}
                 </Collapse.Panel>

datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionButton.tsx

@@ -6,9 +6,10 @@ import {
     useCreateTestConnectionRequestMutation,
     useGetIngestionExecutionRequestLazyQuery,
 } from '../../../../../../graphql/ingestion.generated';
-import { FAILURE, getSourceConfigs, RUNNING, yamlToJson } from '../../../utils';
+import { FAILURE, RUNNING, yamlToJson } from '../../../utils';
 import { TestConnectionResult } from './types';
 import TestConnectionModal from './TestConnectionModal';
+import { SourceConfig } from '../../types';
 
 export function getRecipeJson(recipeYaml: string) {
     // Convert the recipe into it's json representation, and catch + report exceptions while we do it.
@@ -26,21 +27,19 @@ export function getRecipeJson(recipeYaml: string) {
 }
 
 interface Props {
-    type: string;
     recipe: string;
+    sourceConfigs?: SourceConfig;
 }
 
 function TestConnectionButton(props: Props) {
-    const { type, recipe } = props;
+    const { recipe, sourceConfigs } = props;
     const [isLoading, setIsLoading] = useState(false);
     const [isModalVisible, setIsModalVisible] = useState(false);
     const [pollingInterval, setPollingInterval] = useState<null | NodeJS.Timeout>(null);
     const [testConnectionResult, setTestConnectionResult] = useState<null | TestConnectionResult>(null);
     const [createTestConnectionRequest, { data: requestData }] = useCreateTestConnectionRequestMutation();
     const [getIngestionExecutionRequest, { data: resultData, loading }] = useGetIngestionExecutionRequestLazyQuery();
 
-    const sourceConfigs = getSourceConfigs(type);
-
     useEffect(() => {
         if (requestData && requestData.createTestConnectionRequest) {
             const interval = setInterval(

datahub-web-react/src/app/ingest/source/builder/RecipeForm/TestConnection/TestConnectionModal.tsx

@@ -6,8 +6,9 @@ import styled from 'styled-components/macro';
 import { ReactComponent as LoadingSvg } from '../../../../../../images/datahub-logo-color-loading_pendulum.svg';
 import { ANTD_GRAY } from '../../../../../entity/shared/constants';
 import ConnectionCapabilityView from './ConnectionCapabilityView';
-import { SourceConfig } from '../../../conf/types';
 import { CapabilityReport, SourceCapability, TestConnectionResult } from './types';
+import { SourceConfig } from '../../types';
+import useGetSourceLogoUrl from '../../useGetSourceLogoUrl';
 
 const LoadingWrapper = styled.div`
     display: flex;
@@ -84,7 +85,7 @@ const StyledClose = styled(CloseOutlined)`
 interface Props {
     isLoading: boolean;
     testConnectionFailed: boolean;
-    sourceConfig: SourceConfig;
+    sourceConfig?: SourceConfig;
     testConnectionResult: TestConnectionResult | null;
     hideModal: () => void;
 }
@@ -96,15 +97,17 @@ function TestConnectionModal({
     testConnectionResult,
     hideModal,
 }: Props) {
+    const logoUrl = useGetSourceLogoUrl(sourceConfig?.name || '');
+
     return (
         <Modal
             visible
             onCancel={hideModal}
             footer={<Button onClick={hideModal}>Done</Button>}
             title={
                 <ModalHeader style={{ margin: 0 }}>
-                    <SourceIcon alt="source logo" src={sourceConfig.logoUrl} />
-                    {sourceConfig.displayName} Connection Test
+                    <SourceIcon alt="source logo" src={logoUrl} />
+                    {sourceConfig?.displayName} Connection Test
                 </ModalHeader>
             }
             width={750}
@@ -133,8 +136,8 @@ function TestConnectionModal({
                     </ResultsHeader>
                     <ResultsSubHeader>
                         {testConnectionFailed
-                            ? `A connection was not able to be established with ${sourceConfig.displayName}.`
-                            : `A connection was successfully established with ${sourceConfig.displayName}.`}
+                            ? `A connection was not able to be established with ${sourceConfig?.displayName}.`
+                            : `A connection was successfully established with ${sourceConfig?.displayName}.`}
                     </ResultsSubHeader>
                     <Divider />
                     {testConnectionResult?.internal_failure ? (