From 08d35f2f572e88b6638f7c226b9d0213af76a785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20T=C3=B6lle?= Date: Sat, 14 Jun 2025 15:19:34 +0200 Subject: [PATCH] feat: graceful shutdown when CI job is cancelled (#196) By listening on SIGINT and SIGTERM signals we can stop executing as soon as reasonably possible. This helps to avoid uncessary work and stop the job earlier. Right now we have no manual checks for cancelled contexts, and rely on the http client to check for it while making requests. --- cmd/rp/cmd/root.go | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/cmd/rp/cmd/root.go b/cmd/rp/cmd/root.go index 14a5d14..2799e6d 100644 --- a/cmd/rp/cmd/root.go +++ b/cmd/rp/cmd/root.go @@ -1,9 +1,12 @@ package cmd import ( + "context" "log/slog" "os" + "os/signal" "runtime/debug" + "syscall" "time" "github.com/lmittmann/tint" @@ -13,10 +16,12 @@ import ( var logger *slog.Logger var rootCmd = &cobra.Command{ - Use: "rp", - Short: "", - Long: ``, - Version: version(), + Use: "rp", + Short: "", + Long: ``, + Version: version(), + SilenceUsage: true, // Makes it harder to find the actual error + SilenceErrors: true, // We log manually with slog } func version() string { @@ -41,8 +46,33 @@ func version() string { } func Execute() { - err := rootCmd.Execute() + // Behaviour when cancelling jobs: + // + // GitHub Actions: https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/canceling-a-workflow#steps-github-takes-to-cancel-a-workflow-run + // 1. SIGINT + // 2. Wait 7500ms + // 3. SIGTERM + // 4. Wait 2500ms + // 5. SIGKILL + // + // GitLab CI/CD: https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/4446 + // 1. SIGTERM + // 2. Wait ??? + // 3. SIGKILL + // + // We therefore need to listen on SIGINT and SIGTERM + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + go func() { + // Make sure to stop listening on signals after receiving the first signal to hand control of the signal back + // to the runtime. The Go runtime implements a "force shutdown" if the signal is received again. + <-ctx.Done() + logger.InfoContext(ctx, "Received shutdown signal, stopping...") + stop() + }() + + err := rootCmd.ExecuteContext(ctx) if err != nil { + logger.ErrorContext(ctx, err.Error()) os.Exit(1) } }