chore: consolidate archive extraction (CN-941)

lib/extractor/docker-archive/index.ts

@@ -1,24 +1,12 @@
-import { normalize as normalizePath } from "path";
-import { HashAlgorithm } from "../../types";
+import {
+  createGetImageIdFromManifest,
+  dockerArchiveConfig,
+  getManifestLayers,
+} from "../generic-archive-extractor";
 
-import { DockerArchiveManifest } from "../types";
 export { extractArchive } from "./layer";
 
-export function getManifestLayers(manifest: DockerArchiveManifest) {
-  return manifest.Layers.map((layer) => normalizePath(layer));
-}
+export { getManifestLayers };
 
-export function getImageIdFromManifest(
-  manifest: DockerArchiveManifest,
-): string {
-  try {
-    const imageId = manifest.Config.split(".")[0];
-    if (imageId.includes(":")) {
-      // imageId includes the algorithm prefix
-      return imageId;
-    }
-    return `${HashAlgorithm.Sha256}:${imageId}`;
-  } catch (err) {
-    throw new Error("Failed to extract image ID from archive manifest");
-  }
-}
+export const getImageIdFromManifest =
+  createGetImageIdFromManifest(dockerArchiveConfig);

lib/extractor/docker-archive/layer.ts

@@ -1,135 +1,6 @@
-import * as Debug from "debug";
-import { createReadStream } from "fs";
-import * as gunzip from "gunzip-maybe";
-import { basename, normalize as normalizePath } from "path";
-import { Readable } from "stream";
-import { extract, Extract } from "tar-stream";
-import { InvalidArchiveError } from "..";
-import { streamToJson } from "../../stream-utils";
-import { PluginOptions } from "../../types";
-import { extractImageLayer } from "../layer";
 import {
-  DockerArchiveManifest,
-  ExtractAction,
-  ExtractedLayers,
-  ExtractedLayersAndManifest,
-  ImageConfig,
-} from "../types";
+  createExtractArchive,
+  dockerArchiveConfig,
+} from "../generic-archive-extractor";
 
-const debug = Debug("snyk");
-
-/**
- * Retrieve the products of files content from the specified docker-archive.
- * @param dockerArchiveFilesystemPath Path to image file saved in docker-archive format.
- * @param extractActions Array of pattern-callbacks pairs.
- * @param options PluginOptions
- * @returns Array of extracted files products sorted by the reverse order of the layers from last to first.
- */
-export async function extractArchive(
-  dockerArchiveFilesystemPath: string,
-  extractActions: ExtractAction[],
-  _options: Partial<PluginOptions>,
-): Promise<ExtractedLayersAndManifest> {
-  return new Promise((resolve, reject) => {
-    const tarExtractor: Extract = extract();
-    const layers: Record<string, ExtractedLayers> = {};
-    let manifest: DockerArchiveManifest;
-    let imageConfig: ImageConfig;
-
-    tarExtractor.on("entry", async (header, stream, next) => {
-      if (header.type === "file") {
-        const normalizedName = normalizePath(header.name);
-        if (isTarFile(normalizedName)) {
-          try {
-            layers[normalizedName] = await extractImageLayer(
-              stream,
-              extractActions,
-            );
-          } catch (error) {
-            debug(`Error extracting layer content from: '${error.message}'`);
-            reject(new Error("Error reading tar archive"));
-          }
-        } else if (isManifestFile(normalizedName)) {
-          const manifestArray = await getManifestFile<DockerArchiveManifest[]>(
-            stream,
-          );
-          manifest = manifestArray[0];
-        } else if (isImageConfigFile(normalizedName)) {
-          imageConfig = await getManifestFile<ImageConfig>(stream);
-        }
-      }
-
-      stream.resume(); // auto drain the stream
-      next(); // ready for next entry
-    });
-
-    tarExtractor.on("finish", () => {
-      try {
-        resolve(
-          getLayersContentAndArchiveManifest(manifest, imageConfig, layers),
-        );
-      } catch (error) {
-        debug(
-          `Error getting layers and manifest content from docker archive: ${error.message}`,
-        );
-        reject(new InvalidArchiveError("Invalid Docker archive"));
-      }
-    });
-
-    tarExtractor.on("error", (error) => reject(error));
-
-    createReadStream(dockerArchiveFilesystemPath)
-      .pipe(gunzip())
-      .pipe(tarExtractor);
-  });
-}
-
-function getLayersContentAndArchiveManifest(
-  manifest: DockerArchiveManifest,
-  imageConfig: ImageConfig,
-  layers: Record<string, ExtractedLayers>,
-): ExtractedLayersAndManifest {
-  // skip (ignore) non-existent layers
-  // get the layers content without the name
-  // reverse layers order from last to first
-  const layersWithNormalizedNames = manifest.Layers.map((layersName) =>
-    normalizePath(layersName),
-  );
-  const filteredLayers = layersWithNormalizedNames
-    .filter((layersName) => layers[layersName])
-    .map((layerName) => layers[layerName])
-    .reverse();
-
-  if (filteredLayers.length === 0) {
-    throw new Error("We found no layers in the provided image");
-  }
-
-  return {
-    layers: filteredLayers,
-    manifest,
-    imageConfig,
-  };
-}
-
-/**
- * Note: consumes the stream.
- */
-async function getManifestFile<T>(stream: Readable): Promise<T> {
-  return streamToJson<T>(stream);
-}
-
-function isManifestFile(name: string): boolean {
-  return name === "manifest.json";
-}
-
-function isImageConfigFile(name: string): boolean {
-  const configRegex = new RegExp("[A-Fa-f0-9]{64}\\.json");
-  return configRegex.test(name);
-}
-
-function isTarFile(name: string): boolean {
-  // For both "docker save" and "skopeo copy" style archives the
-  // layers are represented as tar archives whose names end in .tar.
-  // For Docker this is "layer.tar", for Skopeo - "<sha256ofLayer>.tar".
-  return basename(name).endsWith(".tar");
-}
+export const extractArchive = createExtractArchive(dockerArchiveConfig);

lib/extractor/generic-archive-extractor.ts

@@ -0,0 +1,167 @@
+import * as Debug from "debug";
+import { createReadStream } from "fs";
+import * as gunzip from "gunzip-maybe";
+import { basename, normalize as normalizePath } from "path";
+import { Readable } from "stream";
+import { extract, Extract } from "tar-stream";
+import { streamToJson } from "../stream-utils";
+
+export class InvalidArchiveError extends Error {
+  constructor(message: string) {
+    super();
+    this.name = "InvalidArchiveError";
+    this.message = message;
+  }
+}
+import { HashAlgorithm, PluginOptions } from "../types";
+import { extractImageLayer } from "./layer";
+import {
+  ExtractAction,
+  ExtractedLayers,
+  ExtractedLayersAndManifest,
+  ImageConfig,
+  TarArchiveManifest,
+} from "./types";
+
+const debug = Debug("snyk");
+
+export interface ArchiveConfig {
+  isLayerFile: (name: string) => boolean;
+  isImageConfigFile: (name: string) => boolean;
+  formatLabel: string;
+  layerErrorType: string;
+  extractImageId: (configValue: string) => string;
+}
+
+export const dockerArchiveConfig: ArchiveConfig = {
+  isLayerFile: (name) => basename(name).endsWith(".tar"),
+  isImageConfigFile: (name) => new RegExp("[A-Fa-f0-9]{64}\\.json").test(name),
+  formatLabel: "Docker",
+  layerErrorType: "tar",
+  extractImageId: (configValue) => configValue.split(".")[0],
+};
+
+export const kanikoArchiveConfig: ArchiveConfig = {
+  isLayerFile: (name) => basename(name).endsWith(".tar.gz"),
+  isImageConfigFile: (name) => new RegExp("sha256:[A-Fa-f0-9]{64}").test(name),
+  formatLabel: "Kaniko",
+  layerErrorType: "tar.gz",
+  extractImageId: (configValue) => configValue,
+};
+
+export function createExtractArchive(
+  config: ArchiveConfig,
+): (
+  archiveFilesystemPath: string,
+  extractActions: ExtractAction[],
+  options: Partial<PluginOptions>,
+) => Promise<ExtractedLayersAndManifest> {
+  return (archiveFilesystemPath, extractActions, _options) =>
+    new Promise((resolve, reject) => {
+      const tarExtractor: Extract = extract();
+      const layers: Record<string, ExtractedLayers> = {};
+      let manifest: TarArchiveManifest;
+      let imageConfig: ImageConfig;
+
+      tarExtractor.on("entry", async (header, stream, next) => {
+        if (header.type === "file") {
+          const normalizedName = normalizePath(header.name);
+          if (config.isLayerFile(normalizedName)) {
+            try {
+              layers[normalizedName] = await extractImageLayer(
+                stream,
+                extractActions,
+              );
+            } catch (error) {
+              debug(`Error extracting layer content from: '${error.message}'`);
+              reject(
+                new Error(`Error reading ${config.layerErrorType} archive`),
+              );
+            }
+          } else if (isManifestFile(normalizedName)) {
+            const manifestArray = await getManifestFile<TarArchiveManifest[]>(
+              stream,
+            );
+            manifest = manifestArray[0];
+          } else if (config.isImageConfigFile(normalizedName)) {
+            imageConfig = await getManifestFile<ImageConfig>(stream);
+          }
+        }
+
+        stream.resume();
+        next();
+      });
+
+      tarExtractor.on("finish", () => {
+        try {
+          resolve(assembleLayersAndManifest(manifest, imageConfig, layers));
+        } catch (error) {
+          debug(
+            `Error getting layers and manifest content from ${config.formatLabel} archive: ${error.message}`,
+          );
+          reject(
+            new InvalidArchiveError(`Invalid ${config.formatLabel} archive`),
+          );
+        }
+      });
+
+      tarExtractor.on("error", (error) => reject(error));
+
+      createReadStream(archiveFilesystemPath)
+        .on("error", (error) => reject(error))
+        .pipe(gunzip())
+        .pipe(tarExtractor);
+    });
+}
+
+function assembleLayersAndManifest(
+  manifest: TarArchiveManifest,
+  imageConfig: ImageConfig,
+  layers: Record<string, ExtractedLayers>,
+): ExtractedLayersAndManifest {
+  const layersWithNormalizedNames = manifest.Layers.map((layerName) =>
+    normalizePath(layerName),
+  );
+  const filteredLayers = layersWithNormalizedNames
+    .filter((layerName) => layers[layerName])
+    .map((layerName) => layers[layerName])
+    .reverse();
+
+  if (filteredLayers.length === 0) {
+    throw new Error("We found no layers in the provided image");
+  }
+
+  return {
+    layers: filteredLayers,
+    manifest,
+    imageConfig,
+  };
+}
+
+async function getManifestFile<T>(stream: Readable): Promise<T> {
+  return streamToJson<T>(stream);
+}
+
+function isManifestFile(name: string): boolean {
+  return name === "manifest.json";
+}
+
+export function createGetImageIdFromManifest(
+  config: ArchiveConfig,
+): (manifest: TarArchiveManifest) => string {
+  return (manifest) => {
+    try {
+      const imageId = config.extractImageId(manifest.Config);
+      if (imageId.includes(":")) {
+        return imageId;
+      }
+      return `${HashAlgorithm.Sha256}:${imageId}`;
+    } catch (err) {
+      throw new Error("Failed to extract image ID from archive manifest");
+    }
+  };
+}
+
+export function getManifestLayers(manifest: TarArchiveManifest): string[] {
+  return manifest.Layers.map((layer) => normalizePath(layer));
+}

lib/extractor/index.ts

@@ -7,7 +7,9 @@ import {
 import { AutoDetectedUserInstructions, ImageType } from "../types";
 import { PluginOptions } from "../types";
 import * as dockerExtractor from "./docker-archive";
+import { InvalidArchiveError } from "./generic-archive-extractor";
 import * as kanikoExtractor from "./kaniko-archive";
+import { isWhitedOutFile } from "./layer";
 import * as ociExtractor from "./oci-archive";
 import {
   DockerArchiveManifest,
@@ -23,13 +25,7 @@ import {
 
 const debug = Debug("snyk");
 
-export class InvalidArchiveError extends Error {
-  constructor(message) {
-    super();
-    this.name = "InvalidArchiveError";
-    this.message = message;
-  }
-}
+export { InvalidArchiveError } from "./generic-archive-extractor";
 class ArchiveExtractor {
   private extractor: Extractor;
   private fileSystemPath: string;
@@ -260,9 +256,7 @@ function layersWithLatestFileModifications(
   return extractedLayers;
 }
 
-export function isWhitedOutFile(filename: string) {
-  return filename.match(/.wh./gm);
-}
+export { isWhitedOutFile } from "./layer";
 
 function isBufferType(type: FileContent): type is Buffer {
   return (type as Buffer).buffer !== undefined;