Add pve-backup-watch 0.0.1
A Go CLI that waits for a Proxmox VE vzdump backup to finish and then restarts the guest. - Take one or more VMIDs and poll the node's vzdump tasks via pvesh - Handle QEMU VMs (qm) and LXC containers (pct): unlock, graceful shutdown with forced stop after 120s, then start - Restart after any result by default; -only-failed restarts only when the backup did not finish OK, and -warn-ok treats warnings as success - Ignore backups that finished before the tool started - Add -dry-run, -timeout, -interval and -node options - Exit with 0, 1 (failure or timeout) or 2 (usage error) - Embed version, commit and build date via ldflags (VERSION file, Makefile) - Add README and CHANGELOG
This commit is contained in:
phil committed
1 parent
b08ce4b8d0
commit
9461ea94a2
6 files changed
+450
-2
No files matched your search
@@ -0,0 +1,276 @@
|
||||
// proxmox-backup-watch
|
||||
//
|
||||
// Waits for the vzdump task of the given VMID(s) to finish, then restarts the
|
||||
// guest (QEMU VM or LXC container). By default it restarts regardless of the
|
||||
// backup result; use -only-failed to restart only when the backup did not
|
||||
// finish OK.
|
||||
//
|
||||
// Usage: proxmox-backup-watch [options] VMID [VMID...]
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Set at build time with -ldflags (see Makefile).
|
||||
var (
|
||||
version = "dev"
|
||||
commit = "none"
|
||||
date = "unknown"
|
||||
)
|
||||
|
||||
type task struct {
|
||||
ID string `json:"id"`
|
||||
StartTime int64 `json:"starttime"`
|
||||
EndTime *int64 `json:"endtime"`
|
||||
Status string `json:"status"`
|
||||
UPID string `json:"upid"`
|
||||
}
|
||||
|
||||
type config struct {
|
||||
node string
|
||||
dryRun bool
|
||||
timeout time.Duration
|
||||
interval time.Duration
|
||||
onlyFailed bool
|
||||
warnOK bool
|
||||
startedAt int64
|
||||
}
|
||||
|
||||
func main() {
|
||||
os.Exit(run())
|
||||
}
|
||||
|
||||
func run() int {
|
||||
cfg := config{startedAt: time.Now().Unix()}
|
||||
|
||||
fs := flag.NewFlagSet("proxmox-backup-watch", flag.ContinueOnError)
|
||||
fs.BoolVar(&cfg.dryRun, "dry-run", false, "show what would be done, don't restart anything")
|
||||
fs.BoolVar(&cfg.dryRun, "n", false, "shorthand for -dry-run")
|
||||
fs.DurationVar(&cfg.timeout, "timeout", 6*time.Hour, "max time to wait per VMID (0 = forever)")
|
||||
fs.DurationVar(&cfg.timeout, "t", 6*time.Hour, "shorthand for -timeout")
|
||||
fs.DurationVar(&cfg.interval, "interval", 30*time.Second, "poll interval")
|
||||
fs.DurationVar(&cfg.interval, "i", 30*time.Second, "shorthand for -interval")
|
||||
fs.BoolVar(&cfg.onlyFailed, "only-failed", false, "restart only if the backup did NOT finish OK")
|
||||
fs.BoolVar(&cfg.onlyFailed, "f", false, "shorthand for -only-failed")
|
||||
fs.BoolVar(&cfg.warnOK, "warn-ok", false, `with -only-failed, treat "WARNINGS: n" as success`)
|
||||
fs.BoolVar(&cfg.warnOK, "w", false, "shorthand for -warn-ok")
|
||||
fs.StringVar(&cfg.node, "node", "", "Proxmox node name (default: short hostname)")
|
||||
var showVersion bool
|
||||
fs.BoolVar(&showVersion, "version", false, "print version and exit")
|
||||
fs.BoolVar(&showVersion, "v", false, "shorthand for -version")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(os.Stderr, "Usage: proxmox-backup-watch [options] VMID [VMID...]")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
|
||||
// Allow flags before, between and after VMIDs.
|
||||
var vmids []string
|
||||
args := os.Args[1:]
|
||||
for {
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if errors.Is(err, flag.ErrHelp) {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
rest := fs.Args()
|
||||
if len(rest) == 0 {
|
||||
break
|
||||
}
|
||||
vmids = append(vmids, rest[0])
|
||||
args = rest[1:]
|
||||
}
|
||||
|
||||
if showVersion {
|
||||
fmt.Printf("proxmox-backup-watch %s (commit %s, built %s)\n", version, commit, date)
|
||||
return 0
|
||||
}
|
||||
|
||||
if len(vmids) == 0 {
|
||||
fmt.Fprintln(os.Stderr, "Error: at least one VMID is required")
|
||||
fs.Usage()
|
||||
return 2
|
||||
}
|
||||
for _, id := range vmids {
|
||||
if !isNumeric(id) {
|
||||
fmt.Fprintf(os.Stderr, "Error: invalid VMID %q\n", id)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.node == "" {
|
||||
h, err := os.Hostname()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: cannot determine hostname: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
cfg.node, _, _ = strings.Cut(h, ".")
|
||||
}
|
||||
|
||||
fmt.Printf("proxmox-backup-watch %s\n", version)
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
failed := false
|
||||
for _, vmid := range vmids {
|
||||
if err := handle(ctx, &cfg, vmid); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "VM %s: %v\n", vmid, err)
|
||||
failed = true
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func handle(ctx context.Context, cfg *config, vmid string) error {
|
||||
fmt.Printf("VM %s: waiting for backup task to finish...\n", vmid)
|
||||
|
||||
t, err := waitForTask(ctx, cfg, vmid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if cfg.onlyFailed {
|
||||
ok := t.Status == "OK" || (cfg.warnOK && strings.HasPrefix(t.Status, "WARNINGS"))
|
||||
if ok {
|
||||
fmt.Printf("VM %s: backup finished (%s), no restart needed\n", vmid, t.Status)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("VM %s: backup finished (%s) -> restarting\n", vmid, t.Status)
|
||||
return restartGuest(ctx, cfg, vmid)
|
||||
}
|
||||
|
||||
// waitForTask polls until the newest vzdump task for vmid (running now, or
|
||||
// finished after this program started) has an end time.
|
||||
func waitForTask(ctx context.Context, cfg *config, vmid string) (*task, error) {
|
||||
var deadline time.Time
|
||||
if cfg.timeout > 0 {
|
||||
deadline = time.Now().Add(cfg.timeout)
|
||||
}
|
||||
|
||||
for {
|
||||
t, err := latestTask(ctx, cfg, vmid)
|
||||
if err != nil {
|
||||
// Transient pvesh/JSON errors shouldn't abort a long wait.
|
||||
fmt.Fprintf(os.Stderr, "VM %s: warning: %v\n", vmid, err)
|
||||
} else if t != nil && t.EndTime != nil {
|
||||
return t, nil
|
||||
}
|
||||
|
||||
if !deadline.IsZero() && time.Now().After(deadline) {
|
||||
return nil, fmt.Errorf("timed out after %s, no restart", cfg.timeout)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(cfg.interval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func latestTask(ctx context.Context, cfg *config, vmid string) (*task, error) {
|
||||
out, err := output(ctx, "pvesh", "get", "/nodes/"+cfg.node+"/tasks",
|
||||
"--typefilter", "vzdump", "--limit", "500", "--output-format", "json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var tasks []task
|
||||
if err := json.Unmarshal(out, &tasks); err != nil {
|
||||
return nil, fmt.Errorf("parsing pvesh output: %w", err)
|
||||
}
|
||||
|
||||
var latest *task
|
||||
for i := range tasks {
|
||||
t := &tasks[i]
|
||||
if t.ID != vmid {
|
||||
continue
|
||||
}
|
||||
// Running (no endtime) or finished after we started.
|
||||
if t.EndTime != nil && *t.EndTime < cfg.startedAt {
|
||||
continue
|
||||
}
|
||||
if latest == nil || t.StartTime > latest.StartTime {
|
||||
latest = t
|
||||
}
|
||||
}
|
||||
return latest, nil
|
||||
}
|
||||
|
||||
func restartGuest(ctx context.Context, cfg *config, vmid string) error {
|
||||
var tool string
|
||||
switch {
|
||||
case exec.CommandContext(ctx, "qm", "status", vmid).Run() == nil:
|
||||
tool = "qm"
|
||||
case exec.CommandContext(ctx, "pct", "status", vmid).Run() == nil:
|
||||
tool = "pct"
|
||||
default:
|
||||
return errors.New("not found on this node")
|
||||
}
|
||||
|
||||
// Clear a leftover 'backup' lock; failure here is not fatal.
|
||||
_ = runCmd(ctx, cfg, tool, "unlock", vmid)
|
||||
|
||||
if err := runCmd(ctx, cfg, tool, "shutdown", vmid, "--timeout", "120", "--forceStop", "1"); err != nil {
|
||||
return fmt.Errorf("shutdown failed: %w", err)
|
||||
}
|
||||
if err := runCmd(ctx, cfg, tool, "start", vmid); err != nil {
|
||||
return fmt.Errorf("start failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCmd(ctx context.Context, cfg *config, name string, args ...string) error {
|
||||
if cfg.dryRun {
|
||||
fmt.Printf("[dry-run] %s %s\n", name, strings.Join(args, " "))
|
||||
return nil
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func output(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String()))
|
||||
}
|
||||
return stdout.Bytes(), nil
|
||||
}
|
||||
|
||||
func isNumeric(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if r < '0' || r > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
Reference in new issue
Block a user