Reduce install size of non-clients by removing unnecessary files from published tarball

[trivikr]

Is your feature request related to a problem? Please describe.

The modular packages in AWS SDK for JavaScript (v3) reduce the bundle size on tree-shaking, but the clients take too much space in node_modules. This affects runtime environments where users do not bundle the SSDK (like AWS Lambda).

Script to compute bundle sizes for non-clients

const { readdirSync, readFileSync, statSync } = require("fs");
const { join } = require("path");
const { spawnSync } = require("child_process");

const { packages } = JSON.parse(readFileSync(join(process.cwd(), "package.json"))).workspaces;

const getAllFiles = (dirPath, arrayOfFiles) => {
  files = readdirSync(dirPath);

  arrayOfFiles = arrayOfFiles || [];

  files.forEach((file) => {
    if (statSync(dirPath + "/" + file).isDirectory()) {
      arrayOfFiles = getAllFiles(dirPath + "/" + file, arrayOfFiles);
    } else {
      arrayOfFiles.push(join(dirPath, "/", file));
    }
  });

  return arrayOfFiles;
};

let totalSizeInMB = 0;

spawnSync("git", ["clean", "-dfx"]);
spawnSync("yarn");
spawnSync("yarn", ["build:all"]);
spawnSync("./node_modules/.bin/lerna", ["run", "downlevel-dts"]);

const packageNameColLength = 50;
const sizeColLength = 15;

console.log(`| ${"Package name".padEnd(packageNameColLength)} | ${"Unpacked size".padEnd(sizeColLength)} |`);
console.log(`|-${"-".padEnd(packageNameColLength, "-")}-|-${"-".padEnd(sizeColLength, "-")}-|`);
packages
  .map((dir) => dir.replace("/*", ""))
  .forEach((workspacesDir) => {
    // Process each workspace in workspace directory
    readdirSync(join(process.cwd(), workspacesDir), { withFileTypes: true })
      .filter((dirent) => dirent.isDirectory())
      .map((dirent) => dirent.name)
      .forEach((workspaceDir) => {
        if (workspaceDir.startsWith("client-") || workspaceDir.startsWith("aws-")) {
          return;
        }
        try {
          // npm pack posts debug messages to stderr
          // Refs: https://github.com/npm/npm/issues/118#issuecomment-325440
          const { stderr } = spawnSync("npm", ["pack", "--dry-run"], {
            cwd: join(process.cwd(), workspacesDir, workspaceDir),
          });
          const packageName = `@aws-sdk/${workspaceDir}`;
          const unpackedSize = stderr
            .toString()
            .match(/unpacked size: ([^\n]*)/)[1]
            .trim();
          console.log(`| ${packageName.padEnd(packageNameColLength)} | ${unpackedSize.padEnd(sizeColLength)} |`);
          const [size, unit] = unpackedSize.split(" ");
          totalSizeInMB += size / (unit === "kB" ? 1024 : 1);
        } catch (error) {
          console.log(`${workspaceDir}: ${error}`);
        }
      });
  });

console.log(`\nTotal size: ${totalSizeInMB.toFixed(2)} MB`);

Bundle sizes of clients

Package name	Unpacked size
@aws-sdk/lib-dynamodb	493.5 kB
@aws-sdk/lib-storage	153.9 kB
@aws-sdk/abort-controller	48.2 kB
@aws-sdk/body-checksum-browser	41.5 kB
@aws-sdk/body-checksum-node	43.1 kB
@aws-sdk/chunked-blob-reader	36.2 kB
@aws-sdk/chunked-blob-reader-native	32.9 kB
@aws-sdk/chunked-stream-reader-node	43.1 kB
@aws-sdk/config-resolver	151.0 kB
@aws-sdk/core-packages-documentation-generator	153.5 kB
@aws-sdk/credential-provider-cognito-identity	190.1 kB
@aws-sdk/credential-provider-env	52.9 kB
@aws-sdk/credential-provider-imds	164.1 kB
@aws-sdk/credential-provider-ini	113.6 kB
@aws-sdk/credential-provider-node	84.5 kB
@aws-sdk/credential-provider-process	66.5 kB
@aws-sdk/credential-provider-sso	134.0 kB
@aws-sdk/credential-provider-web-identity	131.7 kB
@aws-sdk/credential-providers	337.0 kB
@aws-sdk/endpoint-cache	82.1 kB
@aws-sdk/eventstream-handler-node	70.2 kB
@aws-sdk/eventstream-marshaller	179.2 kB
@aws-sdk/eventstream-serde-browser	67.8 kB
@aws-sdk/eventstream-serde-config-resolver	32.8 kB
@aws-sdk/eventstream-serde-node	67.6 kB
@aws-sdk/eventstream-serde-universal	97.2 kB
@aws-sdk/fetch-http-handler	111.0 kB
@aws-sdk/hash-blob-browser	55.2 kB
@aws-sdk/hash-node	48.0 kB
@aws-sdk/hash-stream-node	59.7 kB
@aws-sdk/invalid-dependency	29.0 kB
@aws-sdk/is-array-buffer	30.1 kB
@aws-sdk/karma-credential-loader	47.0 kB
@aws-sdk/md5-js	124.0 kB
@aws-sdk/middleware-apply-body-checksum	66.0 kB
@aws-sdk/middleware-bucket-endpoint	227.1 kB
@aws-sdk/middleware-content-length	68.7 kB
@aws-sdk/middleware-endpoint-discovery	186.3 kB
@aws-sdk/middleware-eventstream	52.9 kB
@aws-sdk/middleware-expect-continue	57.9 kB
@aws-sdk/middleware-header-default	48.6 kB
@aws-sdk/middleware-host-header	49.9 kB
@aws-sdk/middleware-location-constraint	58.2 kB
@aws-sdk/middleware-logger	78.8 kB
@aws-sdk/middleware-retry	239.5 kB
@aws-sdk/middleware-sdk-api-gateway	59.9 kB
@aws-sdk/middleware-sdk-ec2	66.6 kB
@aws-sdk/middleware-sdk-glacier	88.5 kB
@aws-sdk/middleware-sdk-machinelearning	46.2 kB
@aws-sdk/middleware-sdk-rds	77.9 kB
@aws-sdk/middleware-sdk-route53	69.6 kB
@aws-sdk/middleware-sdk-s3	83.6 kB
@aws-sdk/middleware-sdk-s3-control	138.0 kB
@aws-sdk/middleware-sdk-sqs	78.5 kB
@aws-sdk/middleware-sdk-sts	65.3 kB
@aws-sdk/middleware-sdk-transcribe-streaming	131.1 kB
@aws-sdk/middleware-serde	66.8 kB
@aws-sdk/middleware-signing	135.7 kB
@aws-sdk/middleware-ssec	48.5 kB
@aws-sdk/middleware-stack	133.0 kB
@aws-sdk/middleware-user-agent	85.6 kB
@aws-sdk/node-config-provider	112.2 kB
@aws-sdk/node-http-handler	201.2 kB
@aws-sdk/polly-request-presigner	58.1 kB
@aws-sdk/property-provider	92.5 kB
@aws-sdk/protocol-http	75.5 kB
@aws-sdk/querystring-builder	46.4 kB
@aws-sdk/querystring-parser	45.2 kB
@aws-sdk/s3-presigned-post	162.7 kB
@aws-sdk/s3-request-presigner	111.2 kB
@aws-sdk/service-error-classification	49.2 kB
@aws-sdk/sha256-tree-hash	74.6 kB
@aws-sdk/shared-ini-file-loader	65.3 kB
@aws-sdk/signature-v4	383.9 kB
@aws-sdk/smithy-client	368.6 kB
@aws-sdk/types	341.4 kB
@aws-sdk/url-parser	22.0 kB
@aws-sdk/util-arn-parser	31.6 kB
@aws-sdk/util-base64-browser	59.3 kB
@aws-sdk/util-base64-node	41.4 kB
@aws-sdk/util-body-length-browser	34.8 kB
@aws-sdk/util-body-length-node	34.2 kB
@aws-sdk/util-buffer-from	37.2 kB
@aws-sdk/util-create-request	67.7 kB
@aws-sdk/util-credentials	56.0 kB
@aws-sdk/util-dynamodb	138.5 kB
@aws-sdk/util-format-url	49.8 kB
@aws-sdk/util-hex-encoding	39.7 kB
@aws-sdk/util-locate-window	32.1 kB
@aws-sdk/util-uri-escape	36.4 kB
@aws-sdk/util-user-agent-browser	72.9 kB
@aws-sdk/util-user-agent-node	64.8 kB
@aws-sdk/util-utf8-browser	61.4 kB
@aws-sdk/util-utf8-node	34.3 kB
@aws-sdk/util-waiter	77.8 kB
@aws-sdk/xml-builder	55.3 kB

Total size: 9.10 MB

Describe the solution you'd like

Reduce the install size of the clients by taking the following steps:

• Publish only the files in dist folders.
• Remove comments from transpiled *.js files.
• Strip comments from downleveled *.dts files.
• Remove sourceMap files.

Example repo: https://github.com/trivikr/temp-client-s3

Additional context

This is a tracking issue for reducing publish size of non-clients.
The codegen-free part of #2797

Previous issues:

• Move source files to "src" folder in clients #1306
• Use files field to ensure that only critical parts of the packages are shipped to users #1594
• Use files field to ensure that only critical parts of the packages are shipped to users #1594
• "Modularized packages" are still large #2765
• Wasteful Package Size #2132

[github-actions]

This thread has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs and link to relevant comments in this thread.