You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
67 lines
2.0 KiB
Bash
67 lines
2.0 KiB
Bash
3 years ago
|
#!/usr/bin/env bash
|
||
|
# vim:ts=4:sts=4:sw=4:et
|
||
|
#
|
||
|
# Author: Hari Sekhon
|
||
|
# Date: 2021-11-02 17:00:24 +0000 (Tue, 02 Nov 2021)
|
||
|
#
|
||
2 years ago
|
# https://github.com/HariSekhon/DevOps-Bash-tools
|
||
3 years ago
|
#
|
||
|
# License: see accompanying Hari Sekhon LICENSE file
|
||
|
#
|
||
|
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
|
||
|
#
|
||
|
# https://www.linkedin.com/in/HariSekhon
|
||
|
#
|
||
|
|
||
|
set -euo pipefail
|
||
|
[ -n "${DEBUG:-}" ] && set -x
|
||
3 years ago
|
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
3 years ago
|
|
||
1 year ago
|
# shellcheck disable=SC1090,SC1091
|
||
3 years ago
|
. "$srcdir/lib/utils.sh"
|
||
|
|
||
|
# shellcheck disable=SC2034,SC2154
|
||
|
usage_description="
|
||
3 years ago
|
Lists AWS Batch jobs in a given queue older than N hours (default: 24)
|
||
3 years ago
|
|
||
3 years ago
|
Includes jobs stuck in pending and runnable states as these are usually stuck due to an environment/configuration issue if they've been pending for a long time
|
||
|
|
||
3 years ago
|
Useful to find jobs that have become too long running, eg. more than 24 hours, or jobs far exceeding their expected time, including jobs that can get stuck with memory allocation errors on shared VMs
|
||
|
|
||
3 years ago
|
Returns JSON list of jobs for further processing
|
||
3 years ago
|
|
||
|
Requires AWS CLI to be configured and authenticated
|
||
|
"
|
||
|
|
||
|
# used by usage() in lib/utils.sh
|
||
|
# shellcheck disable=SC2034
|
||
|
usage_args="<queue-name> [<hours>]"
|
||
|
|
||
|
help_usage "$@"
|
||
|
|
||
|
min_args 1 "$@"
|
||
|
|
||
|
queue="$1"
|
||
|
hours="${2:-24}"
|
||
|
|
||
|
# doesn't support floats
|
||
|
#millis="$((hours * 3600 * 1000))"
|
||
|
millis="$(bc -l <<< "$hours * 3600 * 1000" | sed 's/\..*$//')"
|
||
|
|
||
|
epoch_millis="$(date +%s)000"
|
||
|
|
||
|
before_epoch_millis="$((epoch_millis - millis))"
|
||
|
|
||
3 years ago
|
# --filters only works on newer versions of CLIv2 so jq processing is more reliable across environments:
|
||
3 years ago
|
#
|
||
|
# https://github.com/aws/aws-cli/issues/6526
|
||
|
#
|
||
3 years ago
|
#aws batch list-jobs --job-queue "$queue" --filters "name=BEFORE_CREATED_AT,values=$before_epoch"
|
||
3 years ago
|
|
||
3 years ago
|
for state in SUBMITTED PENDING RUNNABLE STARTING RUNNING; do
|
||
3 years ago
|
aws batch list-jobs --job-queue "$queue" --job-status "$state" |
|
||
3 years ago
|
jq ".jobSummaryList[] | select(.createdAt <= $before_epoch_millis)"
|
||
|
done |
|
||
3 years ago
|
# slurp items back into an array
|
||
|
jq -s .
|