feat(ingest): lookml - support for git checkout (#5924)

Author: shirshanka

.github/workflows/metadata-ingestion.yml

@@ -52,6 +52,7 @@ jobs:
     env:
       SPARK_VERSION: 3.0.3
       DATAHUB_TELEMETRY_ENABLED: false
+      DATAHUB_LOOKML_GIT_TEST_SSH_KEY: ${{ secrets.DATAHUB_LOOKML_GIT_TEST_SSH_KEY }}
     strategy:
       matrix:
         python-version: ["3.7", "3.10"]

metadata-ingestion/docs/sources/looker/lookml_pre.md

@@ -1,20 +1,53 @@
 ### Pre-requisites
 
+#### [Optional] Create a GitHub Deploy Key
+
+To use LookML ingestion through the UI, or automate github checkout through the cli, you must set up a GitHub deploy key for your Looker GitHub repository. Read [this](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) document for how to set up deploy keys for your Looker git repo. 
+
+In a nutshell, there are three steps: 
+1. Generate a private-public ssh key pair. This will typically generate two files, e.g. looker_datahub_deploy_key (this is the private key) and looker_datahub_deploy_key.pub (this is the public key)
+![Image](https://github.com/datahub-project/static-assets/blob/main/imgs/gitssh/ssh-key-generation.png)
+
+2. Add the public key to your Looker git repo as a deploy key with read access (no need to provision write access). Follow the guide [here](https://docs.github.com/en/developers/overview/managing-deploy-keys#deploy-keys) for that.
+![Image](https://github.com/datahub-project/static-assets/blob/main/imgs/gitssh/git-deploy-key.png)
+
+3. Provision the private key (paste the contents of the file) as a secret on the DataHub Ingestion UI. Give it a handy name like **LOOKER_DATAHUB_DEPLOY_KEY**
+![Image](https://github.com/datahub-project/static-assets/blob/main/imgs/gitssh/datahub-secret.png)
+
 #### [Optional] Create an API key
 
 See the [Looker authentication docs](https://docs.looker.com/reference/api-and-integration/api-auth#authentication_with_an_sdk) for the steps to create a client ID and secret.
 You need to ensure that the API key is attached to a user that has Admin privileges. 
 
 If that is not possible, read the configuration section and provide an offline specification of the `connection_to_platform_map` and the `project_name`.
 
-### Ingestion through UI
+### Ingestion Options
+
+You have 3 options for controlling where your ingestion of LookML is run. 
+- The DataHub UI (recommended for the easiest out-of-the-box experience)
+- As a GitHub Action (recommended to ensure that you have the freshest metadata pushed on change)
+- Using the CLI (scheduled via an orchestrator like Airflow)
+
+Read on to learn more about these options. 
+
+#### Ingestion through UI
 
-Ingestion using lookml connector is not supported through the UI.
-However, you can set up ingestion using a GitHub Action to push metadata whenever your main lookml repo changes.
+To ingest LookML metadata through the UI, you must set up a GitHub deploy key using the instructions in the section [above](#optional-create-a-github-deploy-key). Once that is complete, you can follow the on-screen instructions to set up a LookML source using the Ingestion page.
+
+### Ingestion using GitHub Action
+
+You can set up ingestion using a GitHub Action to push metadata whenever your main lookml repo changes.
+The following sample github action file can be modified to emit LookML metadata whenever there is a change to your repository. This ensures that metadata is already fresh and up to date.
 
 #### Sample GitHub Action
 
 Drop this file into your `.github/workflows` directory inside your Looker github repo.
+You need to set up the following secrets in your GitHub repository to get this workflow to work:
+- DATAHUB_GMS_HOST: The endpoint where your DataHub host is running
+- DATAHUB_TOKEN: An authentication token provisioned for DataHub ingestion
+- LOOKER_BASE_URL: The base url where your Looker assets are hosted (e.g. https://acryl.cloud.looker.com)
+- LOOKER_CLIENT_ID: A provisioned Looker Client ID
+- LOOKER_CLIENT_SECRET: A provisioned Looker Client Secret
 
 ```
 name: lookml metadata upload
@@ -25,12 +58,6 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
-  pull_request:
-    branches:
-      - main
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
   release:
     types: [published, edited]
   workflow_dispatch:
@@ -59,9 +86,9 @@ jobs:
                 branch: ${{ github.ref }}
               # Options
               #connection_to_platform_map:
-              #  acryl-snow: snowflake
-                  #platform: snowflake
-                  #default_db: DEMO_PIPELINE
+              #  connection-name:
+                  #platform: platform-name (e.g. snowflake)
+                  #default_db: default-db-name (e.g. DEMO_PIPELINE)
               api:
                 client_id: ${LOOKER_CLIENT_ID}
                 client_secret: ${LOOKER_CLIENT_SECRET}
@@ -76,7 +103,7 @@ jobs:
         env:
           DATAHUB_GMS_HOST: ${{ secrets.DATAHUB_GMS_HOST }}
           DATAHUB_TOKEN: ${{ secrets.DATAHUB_TOKEN }}
-          LOOKER_BASE_URL: https://acryl.cloud.looker.com  # <--- replace with your Looker base URL
+          LOOKER_BASE_URL: ${{ secrets.LOOKER_BASE_URL }}
           LOOKER_CLIENT_ID: ${{ secrets.LOOKER_CLIENT_ID }}
           LOOKER_CLIENT_SECRET: ${{ secrets.LOOKER_CLIENT_SECRET }}
 ```

metadata-ingestion/docs/sources/looker/lookml_recipe.yml

@@ -1,8 +1,13 @@
 source:
   type: "lookml"
   config:
+    # GitHub Coordinates: Used to check out the repo locally and add github links on the dataset's entity page.
+    github_info:
+      repo: org/repo-name
+      deploy_key_file: ${LOOKER_DEPLOY_KEY_FILE} # file containing the private ssh key for a deploy key for the looker git repo
+
     # Coordinates
-    base_folder: /path/to/model/files
+    # base_folder: /path/to/model/files ## Optional if you are not able to provide a GitHub deploy key
 
     # Options
     api:
@@ -28,10 +33,6 @@ source:
     #     default_schema: DEFAULT_SCHEMA # the default schema configured for this connection
     #     platform_instance: bq_warehouse # optional
     #     platform_env: DEV  # optional
-
-    # Optional additional github information. Used to add github links on the dataset's entity page.
-    github_info:
-      repo: org/repo-name
 # Default sink is datahub-rest and doesn't need to be configured
 # See https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for customization options
 

metadata-ingestion/docs/sources/snowflake/snowflake-usage_recipe.yml

@@ -22,5 +22,6 @@ source:
       deny:
         - "information_schema.*"
 
-sink:
-# sink configs
+# Default sink is datahub-rest and doesn't need to be configured
+# See https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for customization options
+

metadata-ingestion/docs/sources/snowflake/snowflake_recipe.yml

@@ -49,5 +49,5 @@ source:
       deny:
         - '.*information_schema.*'
 
-sink:
-# sink configs
+# Default sink is datahub-rest and doesn't need to be configured
+# See https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for customization options

metadata-ingestion/setup.py

@@ -59,6 +59,7 @@ def get_long_description():
     "cached_property",
     "ijson",
     "click-spinner",
+    "GitPython>2",
 }
 
 kafka_common = {
@@ -135,7 +136,7 @@ def get_long_description():
 
 looker_common = {
     # Looker Python SDK
-    "looker-sdk==22.2.1"
+    "looker-sdk==22.2.1",
 }
 
 bigquery_common = {

metadata-ingestion/src/datahub/configuration/github.py

@@ -1,25 +1,73 @@
-from pydantic import Field, validator
+import os
+from typing import Any, Dict, Optional
+
+from pydantic import Field, FilePath, SecretStr, validator
 
 from datahub.configuration.common import ConfigModel, ConfigurationError
 
 
 class GitHubInfo(ConfigModel):
     repo: str = Field(
-        description="Name of your github repo. e.g. repo for https://github.com/datahub-project/datahub is `datahub-project/datahub`."
+        description="Name of your github repository in org/repo format. e.g. repo for https://github.com/datahub-project/datahub is `datahub-project/datahub`."
     )
     branch: str = Field(
         "main",
         description="Branch on which your files live by default. Typically main or master.",
     )
-    base_url: str = Field("https://github.com", description="Base url for Github")
+    base_url: str = Field(
+        "https://github.com",
+        description="Base url for Github. Used to construct clickable links on the UI.",
+    )
+
+    deploy_key_file: Optional[FilePath] = Field(
+        None,
+        description="A private key file that contains an ssh key that has been configured as a deploy key for this repository. Use a file where possible, else see deploy_key for a config field that accepts a raw string.",
+    )
+    deploy_key: Optional[SecretStr] = Field(
+        None,
+        description="A private key that contains an ssh key that has been configured as a deploy key for this repository. See deploy_key_file if you want to use a file that contains this key.",
+    )
+
+    repo_ssh_locator: Optional[str] = Field(
+        None,
+        description="Auto-inferred from repo as [email protected]:{repo}, but you can override this if needed.",
+    )
 
     @validator("repo")
     def repo_should_be_org_slash_repo(cls, repo: str) -> str:
         if "/" not in repo or len(repo.split("/")) != 2:
             raise ConfigurationError(
-                "github repo should be in organization/repo form e.g. acryldata/datahub-helm"
+                "github repo should be in organization/repo form e.g. datahub-project/datahub"
             )
         return repo
 
     def get_url_for_file_path(self, file_path: str) -> str:
         return f"{self.base_url}/{self.repo}/blob/{self.branch}/{file_path}"
+
+    @validator("deploy_key_file")
+    def deploy_key_file_should_be_readable(
+        cls, v: Optional[FilePath]
+    ) -> Optional[FilePath]:
+        if v is not None:
+            # pydantic does existence checks, we just need to check if we can read it
+            if not os.access(v, os.R_OK):
+                raise ValueError(f"Unable to read deploy key file {v}")
+        return v
+
+    @validator("deploy_key", pre=True, always=True)
+    def deploy_key_filled_from_deploy_key_file(
+        cls, v: Optional[SecretStr], values: Dict[str, Any]
+    ) -> Optional[SecretStr]:
+        if v is None:
+            deploy_key_file = values.get("deploy_key_file")
+            if deploy_key_file is not None:
+                with open(deploy_key_file, "r") as fp:
+                    deploy_key = SecretStr(fp.read())
+                    return deploy_key
+        return v
+
+    @validator("repo_ssh_locator", always=True)
+    def auto_infer_from_repo(cls, v: Optional[str], values: Dict[str, Any]) -> str:
+        if v is None:
+            return f"[email protected]:{values.get('repo')}"
+        return v

metadata-ingestion/src/datahub/ingestion/source/git/__init__.py

@@ -0,0 +1,41 @@
+import logging
+import os
+import pathlib
+from pathlib import Path
+from uuid import uuid4
+
+import git
+from pydantic import SecretStr
+
+logger = logging.getLogger(__name__)
+
+
+class GitClone:
+    def __init__(self, tmp_dir):
+        self.tmp_dir = tmp_dir
+
+    def clone(self, ssh_key: SecretStr, repo_url: str) -> Path:
+        unique_dir = str(uuid4())
+        keys_dir = f"{self.tmp_dir}/{unique_dir}/keys"
+        checkout_dir = f"{self.tmp_dir}/{unique_dir}/checkout"
+        dirs_to_create = [keys_dir, checkout_dir]
+        for d in dirs_to_create:
+            os.makedirs(d, exist_ok=True)
+        git_ssh_identity_file = os.path.join(keys_dir, "ssh_key")
+        with open(
+            git_ssh_identity_file,
+            "w",
+            opener=lambda path, flags: os.open(path, flags, 0o600),
+        ) as fp:
+            fp.write(ssh_key.get_secret_value())
+
+        git_ssh_cmd = f"ssh -i {git_ssh_identity_file}"
+        logger.debug("ssh_command=%s", git_ssh_cmd)
+        logger.info(f"⏳ Cloning repo '{repo_url}', this can take some time...")
+        git.Repo.clone_from(
+            repo_url,
+            checkout_dir,
+            env=dict(GIT_SSH_COMMAND=git_ssh_cmd),
+        )
+        logger.info("✅ Cloning complete!")
+        return pathlib.Path(checkout_dir)

metadata-ingestion/src/datahub/ingestion/source/git/git_import.py

@@ -4,7 +4,9 @@
 import pathlib
 import re
 import sys
+import tempfile
 from dataclasses import dataclass, field as dataclass_field, replace
+from datetime import datetime, timedelta
 from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type
 
 import pydantic
@@ -16,6 +18,7 @@
 import datahub.emitter.mce_builder as builder
 from datahub.configuration import ConfigModel
 from datahub.configuration.common import AllowDenyPattern, ConfigurationError
+from datahub.configuration.github import GitHubInfo
 from datahub.configuration.source_common import EnvBasedSourceConfigBase
 from datahub.emitter.mcp import MetadataChangeProposalWrapper
 from datahub.ingestion.api.common import PipelineContext
@@ -28,6 +31,7 @@
 from datahub.ingestion.api.registry import import_path
 from datahub.ingestion.api.source import Source, SourceReport
 from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.git.git_import import GitClone
 from datahub.ingestion.source.looker.looker_common import (
     LookerCommonConfig,
     LookerExplore,
@@ -139,8 +143,9 @@ def from_looker_connection(
 
 
 class LookMLSourceConfig(LookerCommonConfig):
-    base_folder: pydantic.DirectoryPath = Field(
-        description="Local filepath where the root of the LookML repo lives. This is typically the root folder where the `*.model.lkml` and `*.view.lkml` files are stored. e.g. If you have checked out your LookML repo under `/Users/jdoe/workspace/my-lookml-repo`, then set `base_folder` to `/Users/jdoe/workspace/my-lookml-repo`."
+    base_folder: Optional[pydantic.DirectoryPath] = Field(
+        None,
+        description="Required if not providing github configuration and deploy keys. A pointer to a local directory (accessible to the ingestion system) where the root of the LookML repo has been checked out (typically via a git clone). This is typically the root folder where the `*.model.lkml` and `*.view.lkml` files are stored. e.g. If you have checked out your LookML repo under `/Users/jdoe/workspace/my-lookml-repo`, then set `base_folder` to `/Users/jdoe/workspace/my-lookml-repo`.",
     )
     connection_to_platform_map: Optional[Dict[str, LookerConnectionDefinition]] = Field(
         None,
@@ -225,9 +230,25 @@ def check_either_project_name_or_api_provided(cls, values):
             )
         return values
 
+    @validator("base_folder", always=True)
+    def check_base_folder_if_not_provided(
+        cls, v: Optional[pydantic.DirectoryPath], values: Dict[str, Any]
+    ) -> Optional[pydantic.DirectoryPath]:
+        if v is None:
+            github_info: Optional[GitHubInfo] = values.get("github_info", None)
+            if github_info and github_info.deploy_key:
+                # We have github_info populated correctly, base folder is not needed
+                pass
+            else:
+                raise ValueError(
+                    "base_folder is not provided. Neither has a github deploy_key or deploy_key_file been provided"
+                )
+        return v
+
 
 @dataclass
 class LookMLSourceReport(SourceReport):
+    git_clone_latency: Optional[timedelta] = None
     models_discovered: int = 0
     models_dropped: List[str] = dataclass_field(default_factory=list)
     views_discovered: int = 0
@@ -970,6 +991,7 @@ def _get_upstream_lineage(
             return None
 
     def _get_custom_properties(self, looker_view: LookerView) -> DatasetPropertiesClass:
+        assert self.source_config.base_folder  # this is always filled out
         file_path = str(
             pathlib.Path(looker_view.absolute_file_path).relative_to(
                 self.source_config.base_folder.resolve()
@@ -1063,7 +1085,29 @@ def get_project_name(self, model_name: str) -> str:
                 f"Could not locate a project name for model {model_name}. Consider configuring a static project name in your config file"
             )
 
-    def get_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
+    def get_workunits(self) -> Iterable[MetadataWorkUnit]:
+        if not self.source_config.base_folder:
+            assert self.source_config.github_info
+            # we don't have a base_folder, so we need to clone the repo and process it locally
+            start_time = datetime.now()
+            with tempfile.TemporaryDirectory("lookml_tmp") as tmp_dir:
+                git_clone = GitClone(tmp_dir)
+                # github info deploy key is always populated
+                assert self.source_config.github_info.deploy_key
+                assert self.source_config.github_info.repo_ssh_locator
+                checkout_dir = git_clone.clone(
+                    ssh_key=self.source_config.github_info.deploy_key,
+                    repo_url=self.source_config.github_info.repo_ssh_locator,
+                )
+                self.reporter.git_clone_latency = datetime.now() - start_time
+                self.source_config.base_folder = checkout_dir.resolve()
+                yield from self.get_internal_workunits()
+        else:
+            yield from self.get_internal_workunits()
+
+    def get_internal_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
+        assert self.source_config.base_folder
+
         viewfile_loader = LookerViewFileLoader(
             str(self.source_config.base_folder), self.reporter
         )