Enabled download of includes over HTTP (#605) (#958)

* Enabled download of includes over HTTP

* Added option --git-over-http (default: false)
* When active: Include-Parser uses git spare-checkout instead of git archive

* fixed linter problems

* autodetecting whether sparse checkout should be preferred

* CLI option still available but hidden

* Removed CLI option regardin sparse checkout

* only auto-detection left over

* refactored: parser-includes relies only on git-data

* refactored: git-data detects schema on basis of firecow's proposed regexp

* fix/workaround: test on invalid project includes passes now

* The test attempted to compare an expected error text
  with the assertion error's message.
* The assertion error's message might have changed previously,
  so the test failed
* Analysis of the current logs of failing tests in the pipeline
  shows that different reasons might lead to an expected failure
  when calling 'git archive':
    * the server doesn't exist or isn't reachable (which could timeout), then error message contains
      - ssh: Could not resolve hostname <non-existing-hostname>: Name or service not known
      - fatal: the remote end hung up unexpectedly
    * the credentials for the server are wrong, then error message contains:
      - [email protected]: Permission denied (publickey).
      - fatal: the remote end hung up unexpectedly
    * the repository doesn't exist on the server, then error message contains:
      - ERROR: The project you were looking for could not be found or you don't have permission to view it.
      - fatal: the remote end hung up unexpectedly
    * the file doesn't exist on the server, then error message contains:
        => remote: fatal: pathspec '.gitlab-modue.yml' did not match any files
* Thus it seems OK to test against the 'fatal.*' only

* preparing back-merge as discussed with firecow

* using clearer if clauses for schema default ports

* fixed target path for tar call

---------

Co-authored-by: docdnp <[email protected]>
Co-authored-by: Mads Jon Nielsen <[email protected]>

Author: 3 people

package-lock.json

@@ -59,6 +59,7 @@
     "jest-when": "3.x.x",
     "pkg": "5.x.x",
     "ts-jest": "29.x.x",
+    "ts-node": "^10.9.1",
     "typescript": "5.x.x"
   },
   "engines": {

package.json

@@ -13,6 +13,7 @@ export class GitData {
     };
 
     public readonly remote = {
+        schema: "git",
         port: "22",
         host: "gitlab.com",
         group: "fallback.group",
@@ -66,11 +67,20 @@ export class GitData {
     private async initRemoteData (cwd: string, writeStreams: WriteStreams): Promise<void> {
         try {
             const {stdout: gitRemote} = await Utils.spawn(["git", "remote", "-v"], cwd);
-            const gitRemoteMatch = /.*(?:\/\/|@)(?<host>[^:/]*)(:(?<port>\d+)\/|:|\/)(?<group>.*)\/(?<project>[^ .]+)(?:\.git)?.*/.exec(gitRemote);
+            const gitRemoteMatch = /(?<schema>git|https?)(?::\/\/|@)(?<host>[^:/]*)(:(?<port>\d+)\/|:|\/)(?<group>.*)\/(?<project>[^ .]+)(?:\.git)?.*/.exec(gitRemote);
 
             assert(gitRemoteMatch?.groups != null, "git remote -v didn't provide valid matches");
 
-            this.remote.port = gitRemoteMatch.groups.port ?? "22";
+            this.remote.schema = gitRemoteMatch.groups.schema;
+            if (this.remote.schema === "git") {
+                this.remote.port = gitRemoteMatch.groups.port ?? "22";
+            }
+            if (this.remote.schema === "https") {
+                this.remote.port = gitRemoteMatch.groups.port ?? "443";
+            }
+            if (this.remote.schema === "http") {
+                this.remote.port = gitRemoteMatch.groups.port ?? "80";
+            }
             this.remote.host = gitRemoteMatch.groups.host;
             this.remote.group = gitRemoteMatch.groups.group;
             this.remote.project = gitRemoteMatch.groups.project;

src/git-data.ts

@@ -164,10 +164,28 @@ export class ParserIncludes {
         const remote = gitData.remote;
         const normalizedFile = file.replace(/^\/+/, "");
         try {
-            const target = `${stateDir}/includes/${remote.host}/${project}/${ref}/`;
+            const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`;
             if (await fs.pathExists(`${cwd}/${target}/${normalizedFile}`) && !fetchIncludes) return;
-            await fs.mkdirp(`${cwd}/${target}`);
-            await Utils.bash(`git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${normalizedFile} | tar -f - -xC ${target}`, cwd);
+
+            if (remote.schema.startsWith("http")) {
+                const ext = "tmp-" + Math.random();
+                await fs.ensureFile(`${cwd}/${target}/${normalizedFile}`);
+                await Utils.bash(`
+                    cd ${cwd}/${stateDir} \
+                        && git clone -n --depth=1 --filter=tree:0 \
+                                ${remote.schema}://${remote.host}/${project}.git \
+                                ${cwd}/${target}.${ext} \
+                        && cd ${cwd}/${target}.${ext} \
+                        && git sparse-checkout set --no-cone ${normalizedFile} \
+                        && git checkout \
+                        && cd ${cwd}/${stateDir} \
+                        && cp ${cwd}/${target}.${ext}/${normalizedFile}\
+                              ${cwd}/${target}/${normalizedFile}
+                    `, cwd);
+            } else {
+                await fs.mkdirp(`${cwd}/${target}`);
+                await Utils.bash(`git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${normalizedFile} | tar -f - -xC ${target}/`, cwd);
+            }
         } catch (e) {
             throw new AssertionError({message: `Project include could not be fetched { project: ${project}, ref: ${ref}, file: ${normalizedFile} }\n${e}`});
         }