init
This commit is contained in:
24 files changed
+3822
No files matched your search
@@ -0,0 +1,198 @@
|
||||
package roots
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// CollectResult summarises a targeted garbage collection.
|
||||
type CollectResult struct {
|
||||
// Closure is the number of store paths in the closures of the targets.
|
||||
Closure int `json:"closure"`
|
||||
// Candidates is how many of those were not reachable from any other root.
|
||||
Candidates int `json:"candidates"`
|
||||
// Deleted and Freed come from nix-store's own report.
|
||||
Deleted int `json:"deleted"`
|
||||
Freed int64 `json:"freed"`
|
||||
// Kept lists paths nix refused to delete because they turned out to be
|
||||
// alive (e.g. via keep-derivations or a root created meanwhile).
|
||||
Kept []string `json:"kept,omitempty"`
|
||||
}
|
||||
|
||||
func (c CollectResult) String() string {
|
||||
s := fmt.Sprintf("collected %d store paths, %s freed", c.Deleted, humanBytes(c.Freed))
|
||||
if alive := c.Closure - c.Candidates + len(c.Kept); alive > 0 {
|
||||
s += fmt.Sprintf(" (%d of %d closure paths still alive via other roots)", alive, c.Closure)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
const argChunk = 2000
|
||||
|
||||
// Collect deletes the parts of the targets' closures that are no longer
|
||||
// reachable from any GC root, without walking the whole store the way
|
||||
// `nix-store --gc` does. Liveness is pre-computed from the closure of all
|
||||
// current roots (`--print-roots`); nix-store --delete re-checks it, so a
|
||||
// wrong guess only costs a retry, never a live path.
|
||||
func Collect(ctx context.Context, cfg Config, targets []string) (CollectResult, error) {
|
||||
var res CollectResult
|
||||
targets = existing(targets)
|
||||
if len(targets) == 0 {
|
||||
return res, nil
|
||||
}
|
||||
closure, err := queryClosure(ctx, cfg, targets)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Closure = len(closure)
|
||||
|
||||
out, err := exec.CommandContext(ctx, cfg.NixStoreBin, "--gc", "--print-roots").Output()
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("nix-store --print-roots: %w", cmdError(err))
|
||||
}
|
||||
var rootTargets []string
|
||||
seen := map[string]bool{}
|
||||
sc := bufio.NewScanner(bytes.NewReader(out))
|
||||
sc.Buffer(make([]byte, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
if _, t, ok := strings.Cut(sc.Text(), " -> "); ok && !seen[t] {
|
||||
seen[t] = true
|
||||
rootTargets = append(rootTargets, t)
|
||||
}
|
||||
}
|
||||
live, err := queryClosure(ctx, cfg, existing(rootTargets))
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
|
||||
var cand []string
|
||||
for p := range closure {
|
||||
if !live[p] {
|
||||
cand = append(cand, p)
|
||||
}
|
||||
}
|
||||
res.Candidates = len(cand)
|
||||
|
||||
for attempt := 0; len(cand) > 0 && attempt < 50; attempt++ {
|
||||
n, freed, alive, err := deletePaths(ctx, cfg, cand)
|
||||
res.Deleted += n
|
||||
res.Freed += freed
|
||||
if err != nil && alive == "" {
|
||||
return res, err
|
||||
}
|
||||
if alive == "" {
|
||||
break
|
||||
}
|
||||
res.Kept = append(res.Kept, alive)
|
||||
var next []string
|
||||
for _, p := range existing(cand) {
|
||||
if p != alive {
|
||||
next = append(next, p)
|
||||
}
|
||||
}
|
||||
cand = next
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func existing(paths []string) []string {
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, p := range paths {
|
||||
if p == "" || seen[p] {
|
||||
continue
|
||||
}
|
||||
seen[p] = true
|
||||
if _, err := os.Lstat(p); err == nil {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func queryClosure(ctx context.Context, cfg Config, paths []string) (map[string]bool, error) {
|
||||
res := map[string]bool{}
|
||||
for i := 0; i < len(paths); i += argChunk {
|
||||
args := append([]string{"--query", "--requisites"}, paths[i:min(i+argChunk, len(paths))]...)
|
||||
out, err := exec.CommandContext(ctx, cfg.NixStoreBin, args...).Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nix-store --query --requisites: %w", cmdError(err))
|
||||
}
|
||||
for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
if l != "" {
|
||||
res[l] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var (
|
||||
deletedRe = regexp.MustCompile(`(\d+) store paths deleted, ([0-9.]+) (\w+) freed`)
|
||||
aliveRe = regexp.MustCompile(`[Cc]annot delete path '([^']+)'`)
|
||||
)
|
||||
|
||||
// deletePaths runs nix-store --delete in chunks. If nix refuses a path
|
||||
// because it is alive, that path is returned so the caller can retry
|
||||
// without it.
|
||||
func deletePaths(ctx context.Context, cfg Config, paths []string) (n int, freed int64, alive string, err error) {
|
||||
for i := 0; i < len(paths); i += argChunk {
|
||||
args := append([]string{"--delete"}, paths[i:min(i+argChunk, len(paths))]...)
|
||||
out, cerr := exec.CommandContext(ctx, cfg.NixStoreBin, args...).CombinedOutput()
|
||||
if m := deletedRe.FindSubmatch(out); m != nil {
|
||||
c, _ := strconv.Atoi(string(m[1]))
|
||||
n += c
|
||||
freed += parseBytes(string(m[2]), string(m[3]))
|
||||
}
|
||||
if cerr != nil {
|
||||
if m := aliveRe.FindSubmatch(out); m != nil {
|
||||
return n, freed, string(m[1]), cerr
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
|
||||
return n, freed, "", fmt.Errorf("nix-store --delete: %v: %s", cerr, lines[len(lines)-1])
|
||||
}
|
||||
}
|
||||
return n, freed, "", nil
|
||||
}
|
||||
|
||||
func parseBytes(num, unit string) int64 {
|
||||
f, _ := strconv.ParseFloat(num, 64)
|
||||
mult := map[string]float64{"bytes": 1, "KiB": 1 << 10, "MiB": 1 << 20, "GiB": 1 << 30, "TiB": 1 << 40}[unit]
|
||||
if mult == 0 {
|
||||
mult = 1
|
||||
}
|
||||
return int64(f * mult)
|
||||
}
|
||||
|
||||
func humanBytes(n int64) string {
|
||||
units := []string{"B", "KiB", "MiB", "GiB", "TiB"}
|
||||
f := float64(n)
|
||||
i := 0
|
||||
for f >= 1024 && i < len(units)-1 {
|
||||
f /= 1024
|
||||
i++
|
||||
}
|
||||
if i == 0 {
|
||||
return fmt.Sprintf("%d B", n)
|
||||
}
|
||||
return fmt.Sprintf("%.1f %s", f, units[i])
|
||||
}
|
||||
|
||||
// CollectedTargets returns the store paths of roots whose deletion
|
||||
// succeeded, for passing to Collect.
|
||||
func CollectedTargets(results []Result) []string {
|
||||
var out []string
|
||||
for _, r := range results {
|
||||
if r.OK() && r.Target != "" {
|
||||
out = append(out, r.Target)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in new issue
Block a user