Add manifest-file input (#352)

src/download/download-version.ts

@@ -2,16 +2,17 @@ import { promises as fs } from "node:fs";
 import * as path from "node:path";
 import * as core from "@actions/core";
 import * as tc from "@actions/tool-cache";
-import { Octokit } from "@octokit/core";
-import { paginateRest } from "@octokit/plugin-paginate-rest";
-import { restEndpointMethods } from "@octokit/plugin-rest-endpoint-methods";
 import * as pep440 from "@renovatebot/pep440";
 import * as semver from "semver";
-import { OWNER, REPO, TOOL_CACHE_NAME } from "../utils/constants";
+import {
+  ASTRAL_MIRROR_PREFIX,
+  GITHUB_RELEASES_PREFIX,
+  TOOL_CACHE_NAME,
+  VERSIONS_MANIFEST_URL,
+} from "../utils/constants";
 import type { Architecture, Platform } from "../utils/platforms";
 import { validateChecksum } from "./checksum/checksum";
-
-const PaginatingOctokit = Octokit.plugin(paginateRest, restEndpointMethods);
+import { getAllVersions, getArtifact, getLatestVersion } from "./manifest";
 
 export function tryGetFromToolCache(
   arch: Architecture,
@@ -32,31 +33,46 @@ export async function downloadVersion(
   platform: Platform,
   arch: Architecture,
   version: string,
-  checkSum: string | undefined,
+  checksum: string | undefined,
   githubToken: string,
+  manifestUrl?: string,
 ): Promise<{ version: string; cachedToolDir: string }> {
-  const artifact = `ruff-${arch}-${platform}`;
-  let extension = ".tar.gz";
-  if (platform === "pc-windows-msvc") {
-    extension = ".zip";
-  }
-  const downloadUrl = constructDownloadUrl(version, platform, arch);
-  core.debug(`Downloading ruff from "${downloadUrl}" ...`);
+  const artifact = await getArtifact(version, arch, platform, manifestUrl);
 
-  const downloadPath = await tc.downloadTool(
-    downloadUrl,
-    undefined,
-    githubToken,
+  if (!artifact) {
+    throw new Error(
+      getMissingArtifactMessage(version, arch, platform, manifestUrl),
+    );
+  }
+
+  // For the default astral-sh/versions source, checksum validation relies on
+  // user input or the built-in KNOWN_CHECKSUMS table, not manifest sha256 values.
+  const resolvedChecksum =
+    manifestUrl === undefined
+      ? checksum
+      : resolveChecksum(checksum, artifact.checksum);
+
+  const downloadPath = await downloadArtifact(
+    artifact.downloadUrl,
+    platform,
+    arch,
+    version,
+    getDownloadToken(artifact.downloadUrl, githubToken),
+  );
+  await validateChecksum(
+    resolvedChecksum,
+    downloadPath,
+    arch,
+    platform,
+    version,
   );
-  core.debug(`Downloaded ruff to "${downloadPath}"`);
-  await validateChecksum(checkSum, downloadPath, arch, platform, version);
 
   const extractedDir = await extractDownloadedArtifact(
     version,
     downloadPath,
-    extension,
+    getExtension(platform),
     platform,
-    artifact,
+    `ruff-${arch}-${platform}`,
   );
 
   const cachedToolDir = await tc.cacheDir(
@@ -65,25 +81,60 @@ export async function downloadVersion(
     version,
     arch,
   );
-  return { cachedToolDir, version: version };
+  return { cachedToolDir, version };
 }
 
-function constructDownloadUrl(
-  version: string,
+export function rewriteToMirror(url: string): string | undefined {
+  if (!url.startsWith(GITHUB_RELEASES_PREFIX)) {
+    return undefined;
+  }
+
+  return ASTRAL_MIRROR_PREFIX + url.slice(GITHUB_RELEASES_PREFIX.length);
+}
+
+async function downloadArtifact(
+  downloadUrl: string,
   platform: Platform,
   arch: Architecture,
-): string {
-  const artifactVersionSuffix =
-    semver.lte(version, "v0.4.10") && semver.gte(version, "v0.1.8")
-      ? `-${version}`
-      : "";
-  const artifact = `ruff${artifactVersionSuffix}-${arch}-${platform}`;
-  let extension = ".tar.gz";
-  if (platform === "pc-windows-msvc") {
-    extension = ".zip";
+  version: string,
+  githubToken: string | undefined,
+): Promise<string> {
+  const mirrorUrl = rewriteToMirror(downloadUrl);
+  const resolvedDownloadUrl = mirrorUrl ?? downloadUrl;
+
+  try {
+    return await downloadFile(
+      resolvedDownloadUrl,
+      mirrorUrl !== undefined ? undefined : githubToken,
+    );
+  } catch (err) {
+    if (mirrorUrl === undefined) {
+      throw err;
+    }
+
+    core.warning(
+      `Failed to download from mirror, falling back to GitHub Releases: ${(err as Error).message}`,
+    );
+
+    return await downloadFile(
+      constructDownloadUrl(version, platform, arch),
+      githubToken,
+    );
   }
-  const versionPrefix = semver.lte(version, "v0.4.10") ? "v" : "";
-  return `https://github.com/${OWNER}/${REPO}/releases/download/${versionPrefix}${version}/${artifact}${extension}`;
+}
+
+async function downloadFile(
+  downloadUrl: string,
+  githubToken: string | undefined,
+): Promise<string> {
+  core.info(`Downloading ruff from "${downloadUrl}" ...`);
+  const downloadPath = await tc.downloadTool(
+    downloadUrl,
+    undefined,
+    githubToken,
+  );
+  core.debug(`Downloaded ruff to "${downloadPath}"`);
+  return downloadPath;
 }
 
 async function extractDownloadedArtifact(
@@ -98,7 +149,7 @@ async function extractDownloadedArtifact(
     const fullPathWithExtension = `${downloadPath}${extension}`;
     await fs.copyFile(downloadPath, fullPathWithExtension);
     ruffDir = await tc.extractZip(fullPathWithExtension);
-    // On windows extracting the zip does not create an intermediate directory
+    // On windows extracting the zip does not create an intermediate directory.
   } else {
     ruffDir = await tc.extractTar(downloadPath);
     if (semver.gte(version, "v0.5.0")) {
@@ -113,18 +164,21 @@ async function extractDownloadedArtifact(
 
 export async function resolveVersion(
   versionInput: string,
-  githubToken: string,
+  manifestUrl?: string,
 ): Promise<string> {
   core.debug(`Resolving ${versionInput}...`);
+
   const version =
     versionInput === "latest"
-      ? await getLatestVersion(githubToken)
+      ? await getLatestVersion(manifestUrl)
       : versionInput;
+
   if (tc.isExplicitVersion(version)) {
     core.debug(`Version ${version} is an explicit version.`);
     return version;
   }
-  const availableVersions = await getAvailableVersions(githubToken);
+
+  const availableVersions = await getAvailableVersions(manifestUrl);
   const resolvedVersion = maxSatisfying(availableVersions, version);
   if (resolvedVersion === undefined) {
     throw new Error(`No version found for ${version}`);
@@ -133,77 +187,63 @@ export async function resolveVersion(
   return resolvedVersion;
 }
 
-async function getAvailableVersions(githubToken: string): Promise<string[]> {
-  try {
-    const octokit = new PaginatingOctokit({
-      auth: githubToken,
-    });
-    return await getReleaseTagNames(octokit);
-  } catch (err) {
-    if ((err as Error).message.includes("Bad credentials")) {
-      core.info(
-        "No (valid) GitHub token provided. Falling back to anonymous. Requests might be rate limited.",
-      );
-      const octokit = new PaginatingOctokit();
-      return await getReleaseTagNames(octokit);
-    }
-    throw err;
-  }
+async function getAvailableVersions(manifestUrl?: string): Promise<string[]> {
+  return await getAllVersions(manifestUrl);
 }
 
-async function getReleaseTagNames(
-  octokit: InstanceType<typeof PaginatingOctokit>,
-): Promise<string[]> {
-  const response = await octokit.paginate(octokit.rest.repos.listReleases, {
-    owner: OWNER,
-    repo: REPO,
-  });
-  const releaseTagNames = response.map((release) => release.tag_name);
-  if (releaseTagNames.length === 0) {
-    throw Error(
-      "Github API request failed while getting releases. Check the GitHub status page for outages. Try again later.",
-    );
+function getMissingArtifactMessage(
+  version: string,
+  arch: Architecture,
+  platform: Platform,
+  manifestUrl?: string,
+): string {
+  if (manifestUrl === undefined) {
+    return `Could not find artifact for version ${version}, arch ${arch}, platform ${platform} in ${VERSIONS_MANIFEST_URL} .`;
   }
-  return response.map((release) => release.tag_name);
+
+  return `manifest-file does not contain version ${version}, arch ${arch}, platform ${platform}.`;
 }
 
-async function getLatestVersion(githubToken: string) {
-  const octokit = new PaginatingOctokit({
-    auth: githubToken,
-  });
-
-  let latestRelease: { tag_name: string } | undefined;
-  try {
-    latestRelease = await getLatestRelease(octokit);
-  } catch (err) {
-    if ((err as Error).message.includes("Bad credentials")) {
-      core.info(
-        "No (valid) GitHub token provided. Falling back to anonymous. Requests might be rate limited.",
-      );
-      const octokit = new PaginatingOctokit();
-      latestRelease = await getLatestRelease(octokit);
-    } else {
-      core.error(
-        "Github API request failed while getting latest release. Check the GitHub status page for outages. Try again later.",
-      );
-      throw err;
-    }
-  }
-
-  if (!latestRelease) {
-    throw new Error("Could not determine latest release.");
-  }
-  return latestRelease.tag_name;
+function resolveChecksum(
+  checksum: string | undefined,
+  manifestChecksum: string,
+): string {
+  return checksum !== undefined && checksum !== ""
+    ? checksum
+    : manifestChecksum;
 }
 
-async function getLatestRelease(
-  octokit: InstanceType<typeof PaginatingOctokit>,
-) {
-  const { data: latestRelease } = await octokit.rest.repos.getLatestRelease({
-    owner: OWNER,
-    repo: REPO,
-  });
-  return latestRelease;
+function getDownloadToken(
+  downloadUrl: string,
+  githubToken: string,
+): string | undefined {
+  return downloadUrl.startsWith(GITHUB_RELEASES_PREFIX)
+    ? githubToken
+    : undefined;
+}
+
+function constructDownloadUrl(
+  version: string,
+  platform: Platform,
+  arch: Architecture,
+): string {
+  const normalizedVersion = stripVersionPrefix(version);
+  const artifactVersionSuffix =
+    semver.lte(version, "v0.4.10") && semver.gte(version, "v0.1.8")
+      ? `-${normalizedVersion}`
+      : "";
+  const artifact = `ruff${artifactVersionSuffix}-${arch}-${platform}`;
+  const versionPrefix = semver.lte(version, "v0.4.10") ? "v" : "";
+
+  return `${GITHUB_RELEASES_PREFIX}${versionPrefix}${normalizedVersion}/${artifact}${getExtension(platform)}`;
+}
+
+function stripVersionPrefix(version: string): string {
+  return version.startsWith("v") ? version.slice(1) : version;
+}
+
+function getExtension(platform: Platform): string {
+  return platform === "pc-windows-msvc" ? ".zip" : ".tar.gz";
 }
 
 function maxSatisfying(

src/download/manifest.ts

@@ -0,0 +1,243 @@
+import * as core from "@actions/core";
+import { VERSIONS_MANIFEST_URL } from "../utils/constants";
+import { fetch } from "../utils/fetch";
+import { selectDefaultVariant } from "./variant-selection";
+
+export interface ManifestArtifact {
+  platform: string;
+  variant?: string;
+  url: string;
+  archive_format: string;
+  sha256: string;
+}
+
+export interface ManifestVersion {
+  version: string;
+  artifacts: ManifestArtifact[];
+}
+
+export interface ArtifactResult {
+  archiveFormat: string;
+  checksum: string;
+  downloadUrl: string;
+}
+
+const cachedManifestData = new Map<string, ManifestVersion[]>();
+
+export async function fetchManifest(
+  manifestUrl: string = VERSIONS_MANIFEST_URL,
+): Promise<ManifestVersion[]> {
+  const cachedVersions = cachedManifestData.get(manifestUrl);
+  if (cachedVersions !== undefined) {
+    core.debug(`Using cached manifest data from ${manifestUrl}`);
+    return cachedVersions;
+  }
+
+  core.info(`Fetching manifest data from ${manifestUrl} ...`);
+  const response = await fetch(manifestUrl, {});
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch manifest data: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const body = await response.text();
+  const versions = parseManifest(body, manifestUrl);
+  cachedManifestData.set(manifestUrl, versions);
+  return versions;
+}
+
+export function parseManifest(
+  data: string,
+  sourceDescription: string,
+): ManifestVersion[] {
+  const trimmed = data.trim();
+  if (trimmed === "") {
+    throw new Error(`Manifest at ${sourceDescription} is empty.`);
+  }
+
+  if (trimmed.startsWith("[")) {
+    throw new Error(
+      `Legacy JSON array manifests are no longer supported in ${sourceDescription}. Use the astral-sh/versions manifest format instead.`,
+    );
+  }
+
+  const versions: ManifestVersion[] = [];
+
+  for (const [index, line] of data.split("\n").entries()) {
+    const record = line.trim();
+    if (record === "") {
+      continue;
+    }
+
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(record);
+    } catch (error) {
+      throw new Error(
+        `Failed to parse manifest data from ${sourceDescription} at line ${index + 1}: ${(error as Error).message}`,
+      );
+    }
+
+    if (!isManifestVersion(parsed)) {
+      throw new Error(
+        `Invalid manifest record in ${sourceDescription} at line ${index + 1}.`,
+      );
+    }
+
+    versions.push(parsed);
+  }
+
+  if (versions.length === 0) {
+    throw new Error(`No manifest data found in ${sourceDescription}.`);
+  }
+
+  return versions;
+}
+
+export async function getLatestVersion(
+  manifestUrl: string = VERSIONS_MANIFEST_URL,
+): Promise<string> {
+  const latestVersion = (await fetchManifest(manifestUrl))[0]?.version;
+
+  if (latestVersion === undefined) {
+    throw new Error("No versions found in manifest data");
+  }
+
+  core.debug(`Latest version from manifest: ${latestVersion}`);
+  return latestVersion;
+}
+
+export async function getAllVersions(
+  manifestUrl: string = VERSIONS_MANIFEST_URL,
+): Promise<string[]> {
+  core.info(
+    `Getting available versions from ${manifestSource(manifestUrl)} ...`,
+  );
+  const versions = await fetchManifest(manifestUrl);
+  return versions.map((versionData) => versionData.version);
+}
+
+export async function getArtifact(
+  version: string,
+  arch: string,
+  platform: string,
+  manifestUrl: string = VERSIONS_MANIFEST_URL,
+): Promise<ArtifactResult | undefined> {
+  const versions = await fetchManifest(manifestUrl);
+  const versionData = versions.find((candidate) =>
+    matchesManifestVersion(candidate.version, version),
+  );
+  if (!versionData) {
+    core.debug(`Version ${version} not found in manifest ${manifestUrl}`);
+    return undefined;
+  }
+
+  const targetPlatforms = getTargetPlatforms(
+    versionData.version,
+    arch,
+    platform,
+  );
+  const matchingArtifacts = versionData.artifacts.filter((candidate) =>
+    targetPlatforms.includes(candidate.platform),
+  );
+
+  if (matchingArtifacts.length === 0) {
+    core.debug(
+      `Artifact for ${targetPlatforms.join(" or ")} not found in version ${version}. Available platforms: ${versionData.artifacts
+        .map((candidate) => candidate.platform)
+        .join(", ")}`,
+    );
+    return undefined;
+  }
+
+  const artifact = selectDefaultVariant(
+    matchingArtifacts,
+    `Multiple artifacts found for ${targetPlatforms.join(" or ")} in version ${version}`,
+  );
+
+  return {
+    archiveFormat: artifact.archive_format,
+    checksum: artifact.sha256,
+    downloadUrl: artifact.url,
+  };
+}
+
+export function clearManifestCache(manifestUrl?: string): void {
+  if (manifestUrl === undefined) {
+    cachedManifestData.clear();
+    return;
+  }
+
+  cachedManifestData.delete(manifestUrl);
+}
+
+function manifestSource(manifestUrl: string): string {
+  if (manifestUrl === VERSIONS_MANIFEST_URL) {
+    return VERSIONS_MANIFEST_URL;
+  }
+
+  return `manifest-file ${manifestUrl}`;
+}
+
+function matchesManifestVersion(
+  manifestVersion: string,
+  requestedVersion: string,
+): boolean {
+  return (
+    manifestVersion === requestedVersion ||
+    manifestVersion === withVersionPrefix(requestedVersion)
+  );
+}
+
+function getTargetPlatforms(
+  manifestVersion: string,
+  arch: string,
+  platform: string,
+): string[] {
+  const targetPlatform = `${arch}-${platform}`;
+  const versionPrefixedTargetPlatform = `${stripVersionPrefix(manifestVersion)}-${targetPlatform}`;
+
+  return [targetPlatform, versionPrefixedTargetPlatform];
+}
+
+function withVersionPrefix(version: string): string {
+  return version.startsWith("v") ? version : `v${version}`;
+}
+
+function stripVersionPrefix(version: string): string {
+  return version.startsWith("v") ? version.slice(1) : version;
+}
+
+function isManifestVersion(value: unknown): value is ManifestVersion {
+  if (!isRecord(value)) {
+    return false;
+  }
+
+  if (typeof value.version !== "string" || !Array.isArray(value.artifacts)) {
+    return false;
+  }
+
+  return value.artifacts.every(isManifestArtifact);
+}
+
+function isManifestArtifact(value: unknown): value is ManifestArtifact {
+  if (!isRecord(value)) {
+    return false;
+  }
+
+  const variantIsValid =
+    typeof value.variant === "string" || value.variant === undefined;
+
+  return (
+    typeof value.archive_format === "string" &&
+    typeof value.platform === "string" &&
+    typeof value.sha256 === "string" &&
+    typeof value.url === "string" &&
+    variantIsValid
+  );
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}

src/download/variant-selection.ts

@@ -0,0 +1,39 @@
+interface VariantAwareEntry {
+  variant?: string;
+}
+
+export function selectDefaultVariant<T extends VariantAwareEntry>(
+  entries: T[],
+  duplicateEntryDescription: string,
+): T {
+  const firstEntry = entries[0];
+  if (firstEntry === undefined) {
+    throw new Error("selectDefaultVariant requires at least one candidate.");
+  }
+
+  if (entries.length === 1) {
+    return firstEntry;
+  }
+
+  const defaultEntries = entries.filter((entry) =>
+    isDefaultVariant(entry.variant),
+  );
+  if (defaultEntries.length === 1) {
+    return defaultEntries[0];
+  }
+
+  throw new Error(
+    `${duplicateEntryDescription} with variants ${formatVariants(entries)}. ruff-action currently requires a single default variant for duplicate platform entries.`,
+  );
+}
+
+function isDefaultVariant(variant: string | undefined): boolean {
+  return variant === undefined || variant === "default";
+}
+
+function formatVariants<T extends VariantAwareEntry>(entries: T[]): string {
+  return entries
+    .map((entry) => entry.variant ?? "default")
+    .sort((left, right) => left.localeCompare(right))
+    .join(", ");
+}

src/ruff-action.ts

@@ -11,6 +11,7 @@ import {
   args,
   checkSum,
   githubToken,
+  manifestFile,
   src,
   version,
   versionFile as versionFileInput,
@@ -62,6 +63,7 @@ async function setupRuff(
   githubToken: string,
 ): Promise<{ ruffDir: string; version: string }> {
   const resolvedVersion = await determineVersion();
+  const manifestUrl = manifestFile || undefined;
   if (semver.lt(resolvedVersion, "v0.0.247")) {
     throw Error(
       "This action does not support ruff versions older than 0.0.247",
@@ -82,6 +84,7 @@ async function setupRuff(
     resolvedVersion,
     checkSum,
     githubToken,
+    manifestUrl,
   );
 
   return {
@@ -95,7 +98,7 @@ async function determineVersion(): Promise<string> {
     throw Error("It is not allowed to specify both version and version-file");
   }
   if (version !== "") {
-    return await resolveVersion(version, githubToken);
+    return await resolveVersion(version, manifestFile || undefined);
   }
   if (versionFileInput !== "") {
     const versionFromPyproject =
@@ -105,7 +108,10 @@ async function determineVersion(): Promise<string> {
         `Could not parse version from ${versionFileInput}. Using latest version.`,
       );
     }
-    return await resolveVersion(versionFromPyproject || "latest", githubToken);
+    return await resolveVersion(
+      versionFromPyproject || "latest",
+      manifestFile || undefined,
+    );
   }
   const pyProjectPath = findPyprojectToml(
     src,
@@ -113,7 +119,7 @@ async function determineVersion(): Promise<string> {
   );
   if (!pyProjectPath) {
     core.info(`Could not find pyproject.toml. Using latest version.`);
-    return await resolveVersion("latest", githubToken);
+    return await resolveVersion("latest", manifestFile || undefined);
   }
   const versionFromPyproject =
     getRuffVersionFromRequirementsFile(pyProjectPath);
@@ -122,7 +128,10 @@ async function determineVersion(): Promise<string> {
       `Could not parse version from ${pyProjectPath}. Using latest version.`,
     );
   }
-  return await resolveVersion(versionFromPyproject || "latest", githubToken);
+  return await resolveVersion(
+    versionFromPyproject || "latest",
+    manifestFile || undefined,
+  );
 }
 
 function addRuffToPath(cachedPath: string): void {

src/utils/constants.ts

@@ -1,3 +1,9 @@
 export const REPO = "ruff";
 export const OWNER = "astral-sh";
 export const TOOL_CACHE_NAME = "ruff";
+export const VERSIONS_MANIFEST_URL =
+  "https://raw.githubusercontent.com/astral-sh/versions/main/v1/ruff.ndjson";
+export const GITHUB_RELEASES_PREFIX =
+  "https://github.com/astral-sh/ruff/releases/download/";
+export const ASTRAL_MIRROR_PREFIX =
+  "https://releases.astral.sh/github/ruff/releases/download/";

src/utils/fetch.ts

@@ -0,0 +1,21 @@
+import { ProxyAgent, type RequestInit, fetch as undiciFetch } from "undici";
+
+export function getProxyAgent() {
+  const httpProxy = process.env.HTTP_PROXY || process.env.http_proxy;
+  if (httpProxy) {
+    return new ProxyAgent(httpProxy);
+  }
+
+  const httpsProxy = process.env.HTTPS_PROXY || process.env.https_proxy;
+  if (httpsProxy) {
+    return new ProxyAgent(httpsProxy);
+  }
+
+  return undefined;
+}
+
+export const fetch = async (url: string, opts: RequestInit) =>
+  await undiciFetch(url, {
+    dispatcher: getProxyAgent(),
+    ...opts,
+  });

src/utils/inputs.ts

@@ -6,3 +6,4 @@ export const githubToken = core.getInput("github-token");
 export const args = core.getInput("args");
 export const src = core.getInput("src");
 export const versionFile = core.getInput("version-file");
+export const manifestFile = core.getInput("manifest-file");