actions
/
rust-cache
mirror of https://github.com/Swatinem/rust-cache.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

merge all the caches and simplify

This commit is contained in:
Arpad Borsos 2020-10-03 18:39:38 +02:00
parent f77cb1be47
commit e8e3c57b3b
8 changed files with 197 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
README.md
View File

@ -9,20 +9,12 @@ sensible defaults.
- uses: Swatinem/rust-cache@v1
```
### Registry Cache
### Cache Details
The cache currently caches the following directories:
- `~/.cargo/registry/index`
- `~/.cargo/registry/cache`
This cache is automatically keyed by hashing the `Cargo.lock` / `Cargo.toml`
files. Before persisting, the cache is cleaned of intermediate artifacts and
unneeded dependencies.
**TODO**: The `~/.cargo/git/db` database is not yet persisted, support will be
added at a later point.
### Target Cache
- `./target`
This cache is automatically keyed by:
@ -31,7 +23,13 @@ This cache is automatically keyed by:
- the rustc release / host / hash, and
- a hash of the `Cargo.lock` / `Cargo.toml` files.
Before persisting, the cache is cleaned of anything that is not a needed
dependency. In particular, no caching of workspace crates will be done. For
An additional input `key` can be provided if the builtin keys are not sufficient.
Before persisting, the cache is cleaned of intermediate artifacts and
anything that is not a workspace dependency.
In particular, no caching of workspace crates will be done. For
this reason, this action will automatically set `CARGO_INCREMENTAL=0` to
disable incremental compilation.
**TODO**: The `~/.cargo/git/db` database is not yet persisted, support will be
added at a later point.

4
action.yml
View File

@ -1,9 +1,9 @@
name: "Rust Cache"
description: "A GitHub Action that implements smart caching for rust/cargo projects"
description: "A GitHub Action that implements smart caching for rust/cargo projects with sensible defaults."
author: "Arpad Borsos <arpad.borsos@googlemail.com>"
inputs:
key:
description: "An explicit key for restoring and saving the target cache"
description: "An additional key for the cache"
required: false
runs:
using: "node12"

93
dist/restore/index.js vendored
View File

@ -54612,6 +54612,8 @@ var __asyncValues = (undefined && undefined.__asyncValues) || function (o) {
const stateKey = "RUST_CACHE_KEY";
const stateHash = "RUST_CACHE_HASH";
const home = external_os_default().homedir();
const paths = {
index: external_path_default().join(home, ".cargo/registry/index"),
@ -54623,39 +54625,31 @@ const RefKey = "GITHUB_REF";
function isValidEvent() {
return RefKey in process.env && Boolean(process.env[RefKey]);
}
async function getCaches() {
const rustKey = await getRustKey();
let lockHash = core.getState("lockHash");
async function getCacheConfig() {
let lockHash = core.getState(stateHash);
if (!lockHash) {
lockHash = await getLockfileHash();
core.saveState("lockHash", lockHash);
core.saveState(stateHash, lockHash);
}
let targetKey = core.getInput("key");
if (targetKey) {
targetKey = `${targetKey}-`;
let key = `v0-rust-`;
let inputKey = core.getInput("key");
if (inputKey) {
key += `${inputKey}-`;
}
const job = process.env.GITHUB_JOB;
if (job) {
targetKey = `${job}-${targetKey}`;
key += `${job}-`;
}
const registry = `v0-registry`;
const target = `v0-target-${targetKey}${rustKey}`;
key += await getRustKey();
return {
registry: {
name: "Registry",
paths: [
paths.index,
paths.cache,
],
key: `${registry}-`,
restoreKeys: [registry],
},
target: {
name: "Target",
paths: [paths.target],
key: `${target}-${lockHash}`,
restoreKeys: [target],
},
paths: [
paths.index,
paths.cache,
// TODO: paths.git,
paths.target,
],
key: `${key}-${lockHash}`,
restoreKeys: [key],
};
}
async function getRustKey() {
@ -54680,15 +54674,6 @@ async function getCmdOutput(cmd, args = [], options = {}) {
} }, options));
return stdout;
}
async function getRegistryName() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob();
if (files.length > 1) {
core.warning(`got multiple registries: "${files.join('", "')}"`);
}
const first = files.shift();
return external_path_default().basename(external_path_default().dirname(first));
}
async function getLockfileHash() {
var e_1, _a;
const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false });
@ -54723,30 +54708,26 @@ async function run() {
}
try {
core.exportVariable("CARGO_INCREMENTAL", 0);
const caches = await getCaches();
for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) {
const start = Date.now();
core.startGroup(`Restoring ${name}…`);
core.info(`Restoring paths:\n ${paths.join("\n ")}.`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try {
const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(`CACHEKEY-${type}`, restoreKey);
}
else {
core.info("No cache found.");
}
const start = Date.now();
const { paths, key, restoreKeys } = await getCacheConfig();
core.info(`Restoring paths:\n ${paths.join("\n ")}.`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try {
const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(stateKey, restoreKey);
}
catch (e) {
core.info(`[warning] ${e.message}`);
else {
core.info("No cache found.");
}
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
core.endGroup();
}
catch (e) {
core.info(`[warning] ${e.message}`);
}
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
}
catch (e) {

132
dist/save/index.js vendored
View File

@ -54585,13 +54585,13 @@ var glob = __webpack_require__(8090);
var external_crypto_ = __webpack_require__(6417);
var external_crypto_default = /*#__PURE__*/__webpack_require__.n(external_crypto_);
// EXTERNAL MODULE: ./node_modules/@actions/io/lib/io.js
var io = __webpack_require__(7436);
// EXTERNAL MODULE: external "fs"
var external_fs_ = __webpack_require__(5747);
var external_fs_default = /*#__PURE__*/__webpack_require__.n(external_fs_);
// EXTERNAL MODULE: ./node_modules/@actions/io/lib/io.js
var io = __webpack_require__(7436);
// EXTERNAL MODULE: external "os"
var external_os_ = __webpack_require__(2087);
var external_os_default = /*#__PURE__*/__webpack_require__.n(external_os_);
@ -54615,6 +54615,8 @@ var __asyncValues = (undefined && undefined.__asyncValues) || function (o) {
const stateKey = "RUST_CACHE_KEY";
const stateHash = "RUST_CACHE_HASH";
const home = external_os_default().homedir();
const paths = {
index: external_path_default().join(home, ".cargo/registry/index"),
@ -54626,39 +54628,31 @@ const RefKey = "GITHUB_REF";
function isValidEvent() {
return RefKey in process.env && Boolean(process.env[RefKey]);
}
async function getCaches() {
const rustKey = await getRustKey();
let lockHash = core.getState("lockHash");
async function getCacheConfig() {
let lockHash = core.getState(stateHash);
if (!lockHash) {
lockHash = await getLockfileHash();
core.saveState("lockHash", lockHash);
core.saveState(stateHash, lockHash);
}
let targetKey = core.getInput("key");
if (targetKey) {
targetKey = `${targetKey}-`;
let key = `v0-rust-`;
let inputKey = core.getInput("key");
if (inputKey) {
key += `${inputKey}-`;
}
const job = process.env.GITHUB_JOB;
if (job) {
targetKey = `${job}-${targetKey}`;
key += `${job}-`;
}
const registry = `v0-registry`;
const target = `v0-target-${targetKey}${rustKey}`;
key += await getRustKey();
return {
registry: {
name: "Registry",
paths: [
paths.index,
paths.cache,
],
key: `${registry}-`,
restoreKeys: [registry],
},
target: {
name: "Target",
paths: [paths.target],
key: `${target}-${lockHash}`,
restoreKeys: [target],
},
paths: [
paths.index,
paths.cache,
// TODO: paths.git,
paths.target,
],
key: `${key}-${lockHash}`,
restoreKeys: [key],
};
}
async function getRustKey() {
@ -54683,15 +54677,6 @@ async function getCmdOutput(cmd, args = [], options = {}) {
} }, options));
return stdout;
}
async function getRegistryName() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob();
if (files.length > 1) {
core.warning(`got multiple registries: "${files.join('", "')}"`);
}
const first = files.shift();
return external_path_default().basename(external_path_default().dirname(first));
}
async function getLockfileHash() {
var e_1, _a;
const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false });
@ -54731,50 +54716,35 @@ var save_asyncValues = (undefined && undefined.__asyncValues) || function (o) {
async function run() {
if (!isValidEvent()) {
return;
}
try {
const caches = await getCaches();
let upToDate = true;
for (const [type, { key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) !== key) {
upToDate = false;
break;
}
}
if (upToDate) {
core.info(`All caches up-to-date`);
const start = Date.now();
const { paths: savePaths, key } = await getCacheConfig();
if (core.getState(stateKey) === key) {
core.info(`Cache up-to-date.`);
return;
}
const registryName = await getRegistryName();
const packages = await getPackages();
// TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
await macOsWorkaround();
await io.rmRF(external_path_default().join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
await pruneTarget(packages);
for (const [type, { name, path: paths, key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) === key) {
core.info(`${name} up-to-date.`);
continue;
}
const start = Date.now();
core.startGroup(`Saving ${name}…`);
core.info(`Saving paths:\n ${paths.join("\n ")}.`);
core.info(`Using key "${key}".`);
try {
await cache.saveCache(paths, key);
}
catch (e) {
core.info(`[warning] ${e.message}`);
}
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
core.endGroup();
const registryName = await getRegistryName();
const packages = await getPackages();
await cleanRegistry(registryName, packages);
await cleanTarget(packages);
core.info(`Saving paths:\n ${savePaths.join("\n ")}.`);
core.info(`Using key "${key}".`);
try {
await cache.saveCache(savePaths, key);
}
catch (e) {
core.info(`[warning] ${e.message}`);
}
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
}
catch (e) {
@ -54782,6 +54752,15 @@ async function run() {
}
}
run();
async function getRegistryName() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob();
if (files.length > 1) {
core.warning(`got multiple registries: "${files.join('", "')}"`);
}
const first = files.shift();
return external_path_default().basename(external_path_default().dirname(first));
}
async function getPackages() {
const cwd = process.cwd();
const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"]));
@ -54792,8 +54771,9 @@ async function getPackages() {
return { name: p.name, version: p.version, targets };
});
}
async function pruneRegistryCache(registryName, packages) {
async function cleanRegistry(registryName, packages) {
var e_1, _a;
await io.rmRF(external_path_default().join(paths.index, registryName, ".cache"));
const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
const dir = await external_fs_default().promises.opendir(external_path_default().join(paths.cache, registryName));
try {
@ -54814,7 +54794,7 @@ async function pruneRegistryCache(registryName, packages) {
finally { if (e_1) throw e_1.error; }
}
}
async function pruneTarget(packages) {
async function cleanTarget(packages) {
var e_2, _a;
await external_fs_default().promises.unlink("./target/.rustc_info.json");
await io.rmRF("./target/debug/examples");
@ -54851,7 +54831,7 @@ async function pruneTarget(packages) {
}));
await rmExcept("./target/debug/deps", keepDeps);
}
const twoWeeks = 14 * 24 * 3600 * 1000;
const oneWeek = 7 * 24 * 3600 * 1000;
async function rmExcept(dirName, keepPrefix) {
var e_3, _a;
const dir = await external_fs_default().promises.opendir(dirName);
@ -54865,7 +54845,7 @@ async function rmExcept(dirName, keepPrefix) {
}
const fileName = external_path_default().join(dir.path, dirent.name);
const { mtime } = await external_fs_default().promises.stat(fileName);
if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > twoWeeks) {
if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > oneWeek) {
core.debug(`deleting "${fileName}"`);
if (dirent.isFile()) {
await external_fs_default().promises.unlink(fileName);

2
package.json
View File

@ -2,7 +2,7 @@
"private": true,
"name": "rust-cache",
"version": "1.0.2",
"description": "A GitHub Action that implements smart caching for rust/cargo projects",
"description": "A GitHub Action that implements smart caching for rust/cargo projects with sensible defaults.",
"keywords": [
"actions",
"rust",

77
src/common.ts
View File

@ -6,6 +6,9 @@ import fs from "fs";
import os from "os";
import path from "path";
export const stateKey = "RUST_CACHE_KEY";
const stateHash = "RUST_CACHE_HASH";
const home = os.homedir();
export const paths = {
index: path.join(home, ".cargo/registry/index"),
@ -15,15 +18,9 @@ export const paths = {
};
interface CacheConfig {
name: string;
paths: Array<string>;
key: string;
restoreKeys?: Array<string>;
}
interface Caches {
registry: CacheConfig;
target: CacheConfig;
restoreKeys: Array<string>;
}
const RefKey = "GITHUB_REF";
@ -32,41 +29,36 @@ export function isValidEvent(): boolean {
return RefKey in process.env && Boolean(process.env[RefKey]);
}
export async function getCaches(): Promise<Caches> {
const rustKey = await getRustKey();
let lockHash = core.getState("lockHash");
export async function getCacheConfig(): Promise<CacheConfig> {
let lockHash = core.getState(stateHash);
if (!lockHash) {
lockHash = await getLockfileHash();
core.saveState("lockHash", lockHash);
}
let targetKey = core.getInput("key");
if (targetKey) {
targetKey = `${targetKey}-`;
}
const job = process.env.GITHUB_JOB;
if (job) {
targetKey = `${job}-${targetKey}`;
core.saveState(stateHash, lockHash);
}
const registry = `v0-registry`;
const target = `v0-target-${targetKey}${rustKey}`;
let key = `v0-rust-`;
let inputKey = core.getInput("key");
if (inputKey) {
key += `${inputKey}-`;
}
const job = process.env.GITHUB_JOB;
if (job) {
key += `${job}-`;
}
key += await getRustKey();
return {
registry: {
name: "Registry",
paths: [
paths.index,
paths.cache,
// TODO: paths.git,
],
key: `${registry}-${lockHash}`,
restoreKeys: [registry],
},
target: {
name: "Target",
paths: [paths.target],
key: `${target}-${lockHash}`,
restoreKeys: [target],
},
paths: [
paths.index,
paths.cache,
// TODO: paths.git,
paths.target,
],
key: `${key}-${lockHash}`,
restoreKeys: [key],
};
}
@ -109,17 +101,6 @@ export async function getCmdOutput(
return stdout;
}
export async function getRegistryName(): Promise<string> {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob();
if (files.length > 1) {
core.warning(`got multiple registries: "${files.join('", "')}"`);
}
const first = files.shift()!;
return path.basename(path.dirname(first));
}
async function getLockfileHash(): Promise<string> {
const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false });
const files = await globber.glob();

42
src/restore.ts
View File

@ -1,6 +1,6 @@
import * as cache from "@actions/cache";
import * as core from "@actions/core";
import { getCaches, isValidEvent } from "./common";
import { getCacheConfig, isValidEvent, stateKey } from "./common";
async function run() {
if (!isValidEvent()) {
@ -10,28 +10,26 @@ async function run() {
try {
core.exportVariable("CARGO_INCREMENTAL", 0);
const caches = await getCaches();
for (const [type, { name, paths, key, restoreKeys }] of Object.entries(caches)) {
const start = Date.now();
core.startGroup(`Restoring ${name}…`);
core.info(`Restoring paths:\n ${paths.join("\n ")}.`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try {
const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(`CACHEKEY-${type}`, restoreKey);
} else {
core.info("No cache found.");
}
} catch (e) {
core.info(`[warning] ${e.message}`);
const start = Date.now();
const { paths, key, restoreKeys } = await getCacheConfig();
core.info(`Restoring paths:\n ${paths.join("\n ")}.`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try {
const restoreKey = await cache.restoreCache(paths, key, restoreKeys);
if (restoreKey) {
core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(stateKey, restoreKey);
} else {
core.info("No cache found.");
}
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
core.endGroup();
} catch (e) {
core.info(`[warning] ${e.message}`);
}
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
} catch (e) {
core.info(`[warning] ${e.message}`);

82
src/save.ts
View File

@ -1,10 +1,11 @@
import * as cache from "@actions/cache";
import * as core from "@actions/core";
import * as exec from "@actions/exec";
import * as glob from "@actions/glob";
import * as io from "@actions/io";
import fs from "fs";
import path from "path";
import { getCaches, getCmdOutput, getRegistryName, isValidEvent, paths } from "./common";
import { getCacheConfig, getCmdOutput, isValidEvent, paths, stateKey } from "./common";
async function run() {
if (!isValidEvent()) {
@ -12,49 +13,35 @@ async function run() {
}
try {
const caches = await getCaches();
let upToDate = true;
for (const [type, { key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) !== key) {
upToDate = false;
break;
}
}
if (upToDate) {
core.info(`All caches up-to-date`);
const start = Date.now();
const { paths: savePaths, key } = await getCacheConfig();
if (core.getState(stateKey) === key) {
core.info(`Cache up-to-date.`);
return;
}
const registryName = await getRegistryName();
const packages = await getPackages();
// TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
await macOsWorkaround();
await io.rmRF(path.join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages);
const registryName = await getRegistryName();
const packages = await getPackages();
await pruneTarget(packages);
await cleanRegistry(registryName, packages);
for (const [type, { name, path: paths, key }] of Object.entries(caches)) {
if (core.getState(`CACHEKEY-${type}`) === key) {
core.info(`${name} up-to-date.`);
continue;
}
const start = Date.now();
core.startGroup(`Saving ${name}…`);
core.info(`Saving paths:\n ${paths.join("\n ")}.`);
core.info(`Using key "${key}".`);
try {
await cache.saveCache(paths, key);
} catch (e) {
core.info(`[warning] ${e.message}`);
}
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
core.endGroup();
await cleanTarget(packages);
core.info(`Saving paths:\n ${savePaths.join("\n ")}.`);
core.info(`Using key "${key}".`);
try {
await cache.saveCache(savePaths, key);
} catch (e) {
core.info(`[warning] ${e.message}`);
}
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
} catch (e) {
core.info(`[warning] ${e.message}`);
@ -80,6 +67,17 @@ interface Meta {
}>;
}
async function getRegistryName(): Promise<string> {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
const files = await globber.glob();
if (files.length > 1) {
core.warning(`got multiple registries: "${files.join('", "')}"`);
}
const first = files.shift()!;
return path.basename(path.dirname(first));
}
async function getPackages(): Promise<Packages> {
const cwd = process.cwd();
const meta: Meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"]));
@ -92,7 +90,9 @@ async function getPackages(): Promise<Packages> {
});
}
async function pruneRegistryCache(registryName: string, packages: Packages) {
async function cleanRegistry(registryName: string, packages: Packages) {
await io.rmRF(path.join(paths.index, registryName, ".cache"));
const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
const dir = await fs.promises.opendir(path.join(paths.cache, registryName));
@ -105,12 +105,12 @@ async function pruneRegistryCache(registryName: string, packages: Packages) {
}
}
async function pruneTarget(packages: Packages) {
async function cleanTarget(packages: Packages) {
await fs.promises.unlink("./target/.rustc_info.json");
await io.rmRF("./target/debug/examples");
await io.rmRF("./target/debug/incremental");
let dir: fs.Dir;
let dir: fs.Dir;
// remove all *files* from debug
dir = await fs.promises.opendir("./target/debug");
for await (const dirent of dir) {
@ -137,7 +137,7 @@ async function pruneTarget(packages: Packages) {
await rmExcept("./target/debug/deps", keepDeps);
}
const twoWeeks = 14 * 24 * 3600 * 1000;
const oneWeek = 7 * 24 * 3600 * 1000;
async function rmExcept(dirName: string, keepPrefix: Set<string>) {
const dir = await fs.promises.opendir(dirName);
@ -149,7 +149,7 @@ async function rmExcept(dirName: string, keepPrefix: Set<string>) {
}
const fileName = path.join(dir.path, dirent.name);
const { mtime } = await fs.promises.stat(fileName);
if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > twoWeeks) {
if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > oneWeek) {
core.debug(`deleting "${fileName}"`);
if (dirent.isFile()) {
await fs.promises.unlink(fileName);