fix: don't buffer when extracting binary (#12621)

* chore: pipe tar stream to file

* chore: synchronize internal pipeline

* chore: simulate small VM

* chore: non sudo install test

* chore: pr feedback

* chore: pr feedback

Author: sobolk

.circleci/config.base.yml

@@ -458,6 +458,30 @@ jobs:
             unsetSudoNpmRegistryUrl
             amplify version
 
+  amplify_install_test:
+    <<: *defaults
+    steps:
+      - restore_cache:
+          key: amplify-cli-repo-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          key: amplify-verdaccio-cache-{{ .Branch }}-{{ .Revision }}
+      - restore_cache:
+          key: amplify-pkg-binaries-{{ .Branch }}-{{ .Revision }}
+      - run:
+          name: Start verdaccio and Install Amplify CLI
+          command: |
+            source .circleci/local_publish_helpers.sh
+            startLocalRegistry "$(pwd)/.circleci/verdaccio.yaml"
+            setNpmRegistryUrlToLocal
+            changeNpmGlobalPath
+            # limit memory for new processes to 1GB
+            # this is to make sure that install can work on small VMs
+            # i.e. not buffer content in memory while installing binary
+            ulimit -Sv 1000000
+            npm install -g @aws-amplify/cli
+            unsetNpmRegistryUrl
+            amplify version
+
   amplify_e2e_tests_pkg:
     parameters:
       os:
@@ -1091,6 +1115,17 @@ workflows:
                 - /run-e2e-with-rc\/.*/
                 - /tagged-release\/.*/
                 - /run-e2e\/.*/
+      - amplify_install_test:
+          context: amplify-ecr-image-pull
+          requires:
+            - upload_pkg_binaries
+          filters:
+            branches:
+              only:
+                - dev
+                - /run-e2e-with-rc\/.*/
+                - /tagged-release\/.*/
+                - /run-e2e\/.*/
       - cleanup_resources:
           context:
             - cleanup-resources
@@ -1269,6 +1304,7 @@ workflows:
             - integration_test
             - amplify_e2e_tests_pkg
             - amplify_sudo_install_test
+            - amplify_install_test
             - amplify_console_integration_tests
             - amplify_migration_tests_v10
             - amplify_migration_tests_v6

packages/amplify-cli-npm/binary.ts

@@ -121,7 +121,14 @@ export class Binary {
     console.log(`Downloading release from ${getCompressedBinaryUrl()}`);
     try {
       const res = await axios({ url: getCompressedBinaryUrl(), responseType: 'stream' });
-      await pipeline(res.data, createGunzip(), this.extract());
+      // An array to collect a promises from nested pipeline that extracts tar content to a file.
+      // The tar file to actual file on disk streaming is kicked off by asynchronous events
+      // of extract step. So top level pipeline may complete before streaming is completed.
+      // We capture a Promise from that process to await it before proceeding,
+      // so that we don't call spawnSync prematurely before content streaming completes.
+      const extractPromiseCollector: Array<Promise<void>> = [];
+      await pipeline(res.data, createGunzip(), this.extract(extractPromiseCollector));
+      await Promise.all(extractPromiseCollector);
 
       console.log('amplify has been installed!');
       spawnSync(this.binaryPath, ['version'], { cwd: process.cwd(), stdio: 'inherit' });
@@ -151,29 +158,23 @@ export class Binary {
    *
    * @returns tar.Extract
    */
-  private extract(): tar.Extract {
+  private extract(extractPromiseCollector: Array<Promise<void>>): tar.Extract {
     const extract = tar.extract();
-    const chunks: Uint8Array[] = [];
     extract.on('entry', (header, extractStream, next) => {
       if (header.type === 'file') {
-        extractStream.on('data', (chunk) => {
-          chunks.push(chunk);
+        const fileWriteStream = fs.createWriteStream(this.binaryPath, {
+          mode: 0o755,
         });
+        // pipe tar entry to file stream
+        // and collect a promise so that top level process can await it
+        extractPromiseCollector.push(pipeline(extractStream, fileWriteStream));
       }
       extractStream.on('end', () => {
         next();
       });
 
       extractStream.resume();
     });
-    extract.on('finish', () => {
-      if (chunks.length) {
-        const data = Buffer.concat(chunks);
-        fs.writeFileSync(this.binaryPath, data, {
-          mode: 0o755,
-        });
-      }
-    });
     return extract;
   }
 }