Hari Sekhon 4 years ago
commit 740adfacad

@ -164,10 +164,11 @@ export github=~/github
alias github="sti github; cd $github";
export bitbucket=~/bitbucket
alias bitb='cd $bitbucket'
# clashes with bitbucket-cli
#alias bitbucket='cd $bitbucket'
alias bitb='cd $bitbucket'
alias bb=bitbucket
# used to gitbrowse to bitbucket now in git.sh
#alias bb=bitbucket
if [ -d "$github" ]; then
for x in "$github/"*; do

@ -86,8 +86,10 @@ alias tag="githg tag"
alias um=updatemodules
#type browse &>/dev/null || alias browse=gbrowse
alias gbrowse=gitbrowse
alias gh='gitbrowse "" github'
alias github_actions='gitbrowse actions github'
alias gh='gitbrowse github'
alias gl='gitbrowse gitlab'
alias bb='gitbrowse bitbucket'
alias github_actions='gitbrowse github actions'
alias github_workflows='github_actions'
alias gha='github_actions'
alias ghw='github_workflows'
@ -138,15 +140,21 @@ gitgc(){
}
gitbrowse(){
local url
local filter="${2:-.*}"
url="$(git remote -v | grep "$filter" | awk '/git@|https:/{print $2}' | sed 's,://.*@,://,; s|git@github.com:|https://github.com/| ; s/\.git$//' | head -n1)"
local filter="${1:-.*}"
local path="${2:-}"
local url_base
url_base="$(git remote -v |
grep "$filter" |
awk '/git@|https:/{print $2}' |
head -n1 |
sed 's|git@|https://|; s/\.git$//;' |
perl -pe 's/:(?!\/\/)/\//')"
if [ $# -gt 0 ] &&
[ -z "$url" ]; then
[ -z "$url_base" ]; then
echo "git remote url not found"
return 1
fi
browser "$url/$1"
browser "$url_base/$path"
}
install_git_completion(){

@ -118,6 +118,8 @@ Wercker throttles Shields.io resulting in "invalid response data" - do not use S
<!-- TODO: https://codecov.io, https://coveralls.io -->
[git.io/bash-tools](https://git.io/bash-tools)
400+ DevOps Shell Scripts and Advanced Bash environment.
Heavily used by all my [GitHub repos](https://github.com/harisekhon), dozens of [DockerHub builds](https://hub.docker.com/r/harisekhon) ([Dockerfiles](https://github.com/HariSekhon/Dockerfiles)) and 400+ [CI builds](https://bitbucket.org/harisekhon/devops-bash-tools/src/master/STATUS.md).
@ -233,7 +235,7 @@ etc.
- `gcp_info_compute.sh` - [GCE](https://cloud.google.com/compute/) Virtual Machine instances, [App Engine](https://cloud.google.com/appengine) instances, [Cloud Functions](https://cloud.google.com/functions), [GKE](https://cloud.google.com/kubernetes-engine) clusters, all [Kubernetes](https://kubernetes.io/) objects across all GKE clusters (see `kubernetes_info.sh` below for more details)
- `gcp_info_storage.sh` - [Cloud SQL](https://cloud.google.com/sql) instances, [Cloud Storage](https://cloud.google.com/storage) Buckets, [Cloud Filestore](https://cloud.google.com/filestore), [Cloud Memorystore Redis](https://cloud.google.com/memorystore), [BigTable](https://cloud.google.com/bigtable) clusters and instances, [Datastore](https://cloud.google.com/datastore) indexes
- `gcp_info_networking.sh` - VPC Networks, Addresses, Proxies, Subnets, Routers, Routes, VPN Gateways, VPN Tunnels, Reservations, Firewall rules, Forwarding rules, [Cloud DNS](https://cloud.google.com/dns) managed zones and verified domains
- `gcp_info_bigdata.sh` - [Dataproc](https://cloud.google.com/dataproc) clusters in all regions, [Dataflow](https://cloud.google.com/dataflow) jobs in all regions, [PubSub](https://cloud.google.com/pubsub) messaging topics, [Cloud IOT](https://cloud.google.com/iot-core) registries in all regions
- `gcp_info_bigdata.sh` - [Dataproc](https://cloud.google.com/dataproc) clusters and jobs in all regions, [Dataflow](https://cloud.google.com/dataflow) jobs in all regions, [PubSub](https://cloud.google.com/pubsub) messaging topics, [Cloud IOT](https://cloud.google.com/iot-core) registries in all regions
- `gcp_info_tools.sh` - [Cloud Source Repositories](https://cloud.google.com/source-repositories), [Cloud Builds](https://cloud.google.com/cloud-build), [Container Registry](https://cloud.google.com/container-registry) images across all major repos (`gcr.io`, `us.gcr.io`, `eu.gcr.io`, `asia.gcr.io`), [Deployment Manager](https://cloud.google.com/deployment-manager) deployments
- `gcp_info_auth_config.sh` - Auth Configurations, Organizations & Current Config
- `gcp_info_projects.sh` - Projects names and IDs
@ -244,9 +246,9 @@ etc.
- `gcp_foreach_project.sh` - executes a templated command across all GCP projects, switching `core/project` and replacing `{project_id}` and `{project_name}` in each iteration - powerful, use with care! (used by `gcp_info_all_projects.sh` to call `gcp_info.sh`)
- `gcp_find_orphaned_disks.sh` - lists orphaned disks across all GCP projects (not attached to any compute instance)
- `gcp_secrets_to_kubernetes.sh` - loads [Google Secret Manager](https://cloud.google.com/secret-manager) secrets to Kubernetes secrets in a 1-to-1 mapping. Can specify a list of Google secrets to load or auto-loads all Google secrets with labels `kubernetes-namespace` equalling the current kubectl context's namespace (`kcd` to the right namespace first, see `.bash.d/kubernetes`)
- `gcp_secrets_to_kubernetes_multipart.sh` - creates a Kubernetes secret from multiple Google Secret Manager secrets (used to put `private.pem` and `public.pem` into the same secret to appear as files on disk mounts for apps to use)
- `gcp_secrets_to_kubernetes_multipart.sh` - creates a Kubernetes secret from multiple [Google Secret Manager](https://cloud.google.com/secret-manager) secrets (used to put `private.pem` and `public.pem` into the same secret to appear as files on volume mounts for apps in pods to use)
- `gce_meta.sh` - simple script to query [Google Compute Engine](https://cloud.google.com/compute/) metadata API from within Virtual Machines
- `gce_when_preempted.sh` / `gce_is_preempted.sh` - [Google Compute Engine](https://cloud.google.com/compute/) VM pre-emption latch and boolean check scripts
- `gce_when_preempted.sh` / `gce_is_preempted.sh` - [Google Compute Engine](https://cloud.google.com/compute/) VM preemption latch and boolean check scripts
- `gke_kube_creds.sh` - auto-load all [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine) credentials and contexts for all clusters in the current project so your `kubectl` is ready to rock on GCP
- `curl_auth.sh` - wraps curl to securely load your OAuth API token or username & password from environment variables or interactive starred password prompt through a ram file descriptor to avoid placing them on the command line (which would expose your credentials in the process list or OS audit log files). Used by many other adjacent API querying scripts
- `kubernetes_info.sh` - huge [Kubernetes](https://kubernetes.io/) inventory listing of deployed resources across all namespaces in the current cluster / kube context:
@ -351,6 +353,22 @@ etc.
- `pingdom_checks_average_response_times.sh` - shows the average response times for all Pingdom checks for the last week
- `pingdom_check_latency_by_hour.sh` / `pingdom_checks_latency_by_hour.sh` - shows the average latency for one or all Pingdom checks broken down by hour of the day, over the last week
- `pingdom_sms_credits.sh` - gets the remaining number of Pingdom SMS credits
- `buildkite_*.sh` - [BuildKite](https://buildkite.com/) API scripts:
- `buildkite_pipelines.sh` - list buildkite pipelines for your `$BUILDKITE_ORGANIZATION` / `$BUILDKITE_USER`
- `buildkite_foreach_pipeline.sh` - executes a templated command for each Buildkite pipeline, replacing the `{user}` and `{pipeline}` in each iteration
- `buildkite_agent.sh` - runs a buildkite agent locally on Linux or Mac, or in Docker with choice of Linux distros
- `buildkite_agents.sh` - lists the Buildkite agents connected along with their hostname, IP, started dated and agent details
- `buildkite_pipelines.sh` - lists Buildkite pipelines
- `buildkite_create_pipeline.sh` - create a Buildkite pipeline from a JSON configuration (like from `buildkite_get_pipeline.sh` or `buildkite_save_pipelines.sh`)
- `buildkite_get_pipeline.sh` - gets details for a specific Buildkite pipeline in JSON format
- `buildkite_cancel_scheduled_builds.sh` - cancels BuildKite scheduled builds (to clear a backlog due to offline agents and just focus on new builds)
- `buildkite_rebuild_cancelled_builds.sh` - triggers rebuilds of any cancelled pipelines
- `buildkite_rebuild_failed_builds.sh` - triggers rebuilds of any failed pipelines (useful if you killed an agent and want to re-run them)
- `buildkite_recreate_pipeline.sh` - recreates a pipeline to wipe out all stats (see url and badge caveats in `--help`)
- `buildkite_running_builds.sh` - lists running builds and the agent they're running on
- `buildkite_save_pipelines.sh` - saves all BuildKite pipelines in your `$BUILDKITE_ORGANIZATION` to local JSON files in `$PWD/.buildkite-pipelines/`
- `buildkite_trigger.sh` - triggers BuildKite build job for a given pipeline
- `buildkite_trigger_all.sh` - same as above but for all pipelines
- `jenkins_cli.sh` - runs Jenkins CLI, auto-inferring basic configuations, auto-downloads `jenkins-cli.jar` from Jenkins server if not present, infers a bunch of Jenkins related variables like `$JENKINS_URL` and authentication from `$JENKINS_USER`/`$JENKINS_PASSWORD`, or finds admin password from inside local docker container. Used heavily by `jenkins.sh` one-shot setup
- `jenkins_password.sh` - gets Jenkins admin password from local docker container. Used by `jenkins_cli.sh`
- `jenkins.sh` - one-touch [Jenkins CI](https://jenkins.io/), launches in docker, installs plugins, validates `Jenkinsfile`, configures jobs from `$PWD/setup/jenkins-job.xml` and sets Pipeline to git remote origin's `Jenkinsfile`, triggers build, tails results in terminal. Call from any repo top level directory with a `Jenkinsfile` pipeline and `setup/jenkins-job.xml` (all mine have it)

@ -39,7 +39,7 @@ eg.
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<cmd>"
usage_args="<command> <args>"
help_usage "$@"

@ -18,6 +18,11 @@
# see /usr/local/etc/buildkite-agent/buildkite-agent.cfg for config on Mac
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
@ -30,12 +35,6 @@ BUILDKITE_DOCKER set to any value to use Docker agent regardless
BUILDKITE_DOCKER_TAG set to BuildKite docker agent tag, eg. centos, ubuntu, alpine (default agent latest tag is alpine)
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
help_usage "$@"
# ============================================================================ #

@ -13,25 +13,35 @@
# https://www.linkedin.com/in/harisekhon
#
# https://buildkite.com/docs/apis/rest-api/agents
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_description="
# Lists BuildKite Agents via BuildKite API
#
# https://buildkite.com/docs/apis/rest-api/agents
Lists BuildKite Agents via the BuildKite API
Output format:
<hostname> <ip_address> <started_date> <user_agent>
eg.
myhost.local x.x.x.x 2020-09-06T09:52:51.969Z buildkite-agent/3.20.0.3264 (darwin; amd64)
f805f651ccfe x.x.x.x 2020-09-07T09:41:37.603Z buildkite-agent/3.22.1.x (linux; amd64)
(this second one is running in Docker hence the funny hostname)
"
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
BUILDKITE_ORGANIZATION="${BUILDKITE_ORGANIZATION:-${BUILDKITE_USER:-}}"
check_env_defined BUILDKITE_ORGANIZATION

@ -14,10 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="/path [<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
@ -46,11 +47,9 @@ buildkite_api.sh organizations/hari-sekhon/agents | jq
buildkite_api.sh organizations/hari-sekhon/emojis | jq
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="/path [<curl_options>]"
if [ -z "${BUILDKITE_TOKEN:-}" ]; then
usage "BUILDKITE_TOKEN environment variable is not set (generate this from the Web UI -> Personal Settings -> API Access Tokens (https://buildkite.com/user/api-access-tokens)"
@ -62,8 +61,16 @@ fi
help_usage "$@"
url_path="${1##/}"
shift
min_args 1 "$@"
url_base="https://api.buildkite.com/v2"
api_version=2
url_path="$1"
shift || :
url_path="${url_path#$url_base}"
url_path="${url_path##/}"
if is_curl_min_version 7.55; then
# hide token from process list if curl version is new enough to support this trick

@ -14,10 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
@ -29,11 +30,11 @@ API pagination not returning all results or even indicating if there are more re
https://buildkite.com/docs/apis/rest-api/builds
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
help_usage "$@"
"$srcdir/buildkite_api.sh" 'builds?state=scheduled' "$@" |
jq -r '.[] | [.pipeline.slug, .number, .url] | @tsv' |

@ -14,10 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="pipeline.json [<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
@ -28,11 +29,9 @@ This JSON file can be created from a configuration downloaded by buildkite_get_p
Used by buildkite_recreate_pipeline.sh to wipe out old history and reset stats
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="pipeline.json [<curl_options>]"
# will attempt to parse from pipeline JSON if not given
# remember to set this eg. BUILDKITE_ORGANIZATION="hari-sekhon"

@ -0,0 +1,66 @@
#!/usr/bin/env bash
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-04-01 18:59:00 +0100 (Wed, 01 Apr 2020)
#
# https://github.com/harisekhon/bash-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
# https://www.linkedin.com/in/harisekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_description="
Run a command for each Buildkite pipeline
All arguments become the command template
The command template replaces the following for convenience in each iteration:
{username}, {user} => \$BUILDKITE_ORGANIZATION / \$BUILDKITE_USER
{organization}, {org} => \$BUILDKITE_ORGANIZATION / \$BUILDKITE_USER
{name} => the pipeline name without the user/organization prefix
{pipeline} => the pipeline name with the user/organization prefix
eg.
${0##*/} echo user={user} name={name} pipeline={pipeline}
"
# shellcheck disable=SC2034
usage_args="[<command>]"
help_usage "$@"
min_args 1 "$@"
cmd_template="$*"
# remember to set this eg. BUILDKITE_ORGANIZATION="hari-sekhon"
prefix="${BUILDKITE_ORGANIZATION:-${BUILDKITE_USER:-}}"
while read -r name; do
pipeline="$prefix/$name"
echo "# ============================================================================ #" >&2
echo "# $pipeline" >&2
echo "# ============================================================================ #" >&2
cmd="$cmd_template"
cmd="${cmd//\{username\}/$prefix}"
cmd="${cmd//\{user\}/$prefix}"
cmd="${cmd//\{organization\}/$prefix}"
cmd="${cmd//\{org\}/$prefix}"
cmd="${cmd//\{pipeline\}/$pipeline}"
cmd="${cmd//\{name\}/$name}"
eval "$cmd"
done < <("$srcdir/buildkite_pipelines.sh")

@ -14,10 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<pipeline> [<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
@ -33,11 +34,9 @@ the webhook URL to triggers builds and could lead to a DoS exploit if publicly d
Used by buildkite_recreate_pipeline.sh to wipe out old history and reset stats
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<pipeline> [<curl_options>]"
# remember to set this eg. BUILDKITE_ORGANIZATION="hari-sekhon"
BUILDKITE_ORGANIZATION="${BUILDKITE_ORGANIZATION:-${BUILDKITE_USER:-}}"

@ -14,10 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
@ -28,11 +29,9 @@ eg. trigger a build of each pipeline:
./buildkite_pipelines.sh | while read pipeline; do ./buildkite_trigger.sh \"\$pipeline\"; done
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
# remember to set this eg. BUILDKITE_ORGANIZATION="hari-sekhon"
BUILDKITE_ORGANIZATION="${BUILDKITE_ORGANIZATION:-${BUILDKITE_USER:-}}"

@ -14,6 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
@ -26,12 +31,6 @@ https://buildkite.com/docs/apis/rest-api/builds
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
help_usage "$@"
"$srcdir/buildkite_api.sh" "builds?state=canceled" "$@" |

@ -0,0 +1,61 @@
#!/usr/bin/env bash
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2020-04-01 18:59:00 +0100 (Wed, 01 Apr 2020)
#
# https://github.com/harisekhon/bash-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
# https://www.linkedin.com/in/harisekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_description="
Rebuilds the last N failed builds in BuildKite via its API
Useful to retrying N builds across projects where they may have failed due to agent problems
https://buildkite.com/docs/apis/rest-api/builds
"
# shellcheck disable=SC2034
usage_args="<pipeline> [<num_builds>]"
help_usage "$@"
min_args 1 "$@"
pipeline="$1"
num="${2:-10}"
if ! is_int "$num"; then
usage "num builds must be an integer"
fi
if [ "$num" -lt 1 ] || [ "$num" -gt 100 ]; then
usage "num builds must be an integer between 1 and 100"
fi
# remember to set this eg. BUILDKITE_ORGANIZATION="hari-sekhon"
user_org="${BUILDKITE_ORGANIZATION:-${BUILDKITE_USER:-}}"
"$srcdir/buildkite_api.sh" "/organizations/$user_org/pipelines/$pipeline/builds?state=failed&per_page=$num" |
jq -r '.[] | [.pipeline.slug, .number, .url] | @tsv' |
while read -r name number url; do
echo -n "Rebuilding $name build number $number: "
"$srcdir/buildkite_api.sh" "$url/rebuild" -X PUT |
jq -r '.state'
done

@ -14,6 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
@ -28,12 +33,6 @@ May fail with Forbidden if your trial account has expired (renew or contact supp
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
BUILDKITE_ORGANIZATION="${BUILDKITE_ORGANIZATION:-${BUILDKITE_USER:-}}"
check_env_defined BUILDKITE_ORGANIZATION

@ -14,10 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<pipeline> [<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
@ -42,11 +43,9 @@ buildkite_api.sh organizations | jq -r '.[].slug'
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<pipeline> [<curl_options>]"
help_usage "$@"

@ -14,6 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
@ -26,12 +31,13 @@ https://buildkite.com/docs/apis/rest-api/builds
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
help_usage "$@"
"$srcdir/buildkite_api.sh" 'builds?state=running' "$@"
"$srcdir/buildkite_api.sh" 'builds?state=running' "$@" |
jq -r '.[] | [.pipeline.slug, .branch, .number, .commit, .created_at, .jobs[0].agent.name] | @tsv' |
#while read -r name branch number commit created agent; do
# commit="${commit:0:8}"
# echo "$name $branch $number $commit $created $agent"
#done |
sed 's/\([[:space:]][[:alnum:]]\{8\}\)[[:alnum:]]\{32\}[[:space:]]/\1 /' |
column -t

@ -14,23 +14,22 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
save_dir=".buildkite-pipelines"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
Saves all BuildKite pipelines in your \$BUILDKITE_ORGANIZATION to local JSON files in \$PWD/$save_dir
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
save_dir=".buildkite-pipelines"
help_usage "$@"

@ -14,21 +14,20 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<pipeline> [<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
Triggers BuildKite job for a pipeline given as argument
Triggers a BuildKite build job for a given pipeline
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<pipeline> [<curl_options>]"
# remember to set this eg. BUILDKITE_ORGANIZATION="hari-sekhon"
BUILDKITE_ORGANIZATION="${BUILDKITE_ORGANIZATION:-${BUILDKITE_USER:-}}"

@ -14,21 +14,20 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
Triggers BuildKite jobs for all pipelines in the \$BUILDKITE_ORGANIZATION
Triggers BuildKite build jobs for all pipelines in the \$BUILDKITE_ORGANIZATION
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<curl_options>]"
help_usage "$@"

@ -14,10 +14,11 @@
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="pipeline.json [<curl_options>]"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034
usage_description="
@ -28,11 +29,9 @@ This JSON file can be created from a configuration downloaded by buildkite_get_p
Used by buildkite_recreate_pipeline.sh to wipe out old history and reset stats
"
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "$0")"
# shellcheck disable=SC1090
. "$srcdir/lib/utils.sh"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="pipeline.json [<curl_options>]"
# will attempt to parse from pipeline JSON if not given
# remember to set this eg. BUILDKITE_ORGANIZATION="hari-sekhon"

@ -38,7 +38,7 @@ eg.
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<cmd>"
usage_args="<command> <args>"
help_usage "$@"

@ -38,7 +38,7 @@ eg.
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<cmd>"
usage_args="<command> <args>"
help_usage "$@"

@ -49,7 +49,7 @@ For a more useful example, see:
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<command>"
usage_args="<command> <args>"
help_usage "$@"

@ -95,6 +95,7 @@ Lists in this order (categories broadly reflect the GCP Console grouping of serv
- DNS managed zones & verified domains
- Big Data:
- Dataproc clusters (all regions)
- Dataproc jobs (all regions)
- Dataflow jobs (all regions)
- PubSub topics
- Cloud IOT Registries (all regions)

@ -30,10 +30,16 @@ Lists GCP Big Data resources deployed in the current GCP Project
Lists in this order:
- Dataproc clusters (all regions)
- Dataproc jobs (all regions)
- Dataflow jobs (all regions)
- PubSub topics
- Cloud IOT registries (all regions)
Environment variables of regions to shortcut scanning all regions, comma or space separated:
GCE_REGIONS - for Dataproc clusters and jobs
IOT_REGIONS - for Cloud IOT registries
$gcp_info_formatting_help
"
@ -57,9 +63,36 @@ cat <<EOF
EOF
if is_service_enabled dataproc.googleapis.com; then
gcp_info "Dataproc clusters" gcloud dataproc clusters list --region all
# because --region=all doesn't work
# inherit gce_regions if set elsewhere, eg. gcp_info_compute.sh called first when running gcp_info.sh
gce_regions="${GCE_REGIONS:-${gce_regions:-$(gcloud compute regions list --format='table[no-heading](name)')}}"
gce_regions="${gce_regions//,/ }"
gcp_info "Dataproc clusters: global" gcloud dataproc clusters list --region="global"
for region in $gce_regions; do
gcp_info "Dataproc clusters: $region" gcloud dataproc clusters list --region="$region"
done
else
echo "Dataproc API (dataproc.googleapis.com) is not enabled, skipping..."
fi
# Dataproc jobs
cat <<EOF
gcp_info "Dataproc jobs" gcloud dataproc jobs list --region all
# ============================================================================ #
# D a t a p r o c J o b s
# ============================================================================ #
EOF
if is_service_enabled dataproc.googleapis.com; then
# because --region=all doesn't work
# re-use gce_regions from above
gcp_info "Dataproc jobs: global" gcloud dataproc jobs list --region="global"
for region in $gce_regions; do
gcp_info "Dataproc jobs: $region" gcloud dataproc jobs list --region="$region"
done
else
echo "Dataproc API (dataproc.googleapis.com) is not enabled, skipping..."
fi
@ -80,7 +113,9 @@ EOF
# DISABLED dataflow.googleapis.com Dataflow API
#
#if is_service_enabled dataflow.googleapis.com; then
gcp_info "Dataflow jobs" gcloud dataflow jobs list --region=all
# --region=all actually works here unlike dataproc and cloud iot
# --status=active to see only running jobs
gcp_info "Dataflow jobs" gcloud dataflow jobs list --region=all --status=all
#else
# echo "Dataflow API (dataflow.googleapis.com) is not enabled, skipping..."
#fi
@ -113,8 +148,21 @@ cat <<EOF
EOF
#iot_supported_regions="
#asia-east1
#europe-west1
#us-central1
#"
# get dynamically in case they add a region
# ERROR: (gcloud.iot.registries.list) NOT_FOUND: The cloud region 'projects/$GOOGLE_PROJECT_ID/locations/all' (location 'all') isn't supported. Valid regions: {asia-east1,europe-west1,us-central1}
iot_regions="${IOT_REGIONS:-$(gcloud iot registries list --region="all" 2>&1 | sed 's/.*{//; s/}//; s/,/ /g' || :)}"
iot_regions="${iot_regions//,/ }"
if is_service_enabled cloudiot.googleapis.com; then
gcp_info "Cloud IOT registries" gcloud iot registries list --region=all
for region in $iot_regions; do
gcp_info "Cloud IOT registries: $region" gcloud iot registries list --region="$region"
done
else
echo "Cloud IOT API ( cloudiot.googleapis.com) is not enabled, skipping..."
fi

@ -36,7 +36,7 @@ usage_description="Runs any arguments as a command against each file with a Git
The filename will be appended to the end of each command in each iteration"
# shellcheck disable=SC2034
usage_args="<cmd> <args>"
usage_args="<command> <args>"
help_usage "$@"

@ -51,7 +51,7 @@ or more usefully when chained with the other adjacent github_*.sh / gitlab_*.sh
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<cmd>"
usage_args="<command> <args>"
help_usage "$@"

@ -42,7 +42,7 @@ eg.
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<cmd>"
usage_args="<command> <args>"
help_usage "$@"

@ -40,7 +40,7 @@ eg.
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<cmd>"
usage_args="<command> <args>"
help_usage "$@"

@ -41,6 +41,10 @@ eg.
Since lab contexts like Docker Desktop, Minikube etc are often offline and likely to hang, they are skipped. Deleted GKE clusters you'll need to remove from your kubeconfig yourself before calling this
"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<command> <args>"
help_usage "$@"
min_args 1 "$@"

@ -43,7 +43,7 @@ For real usage examples, see:
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<cmd>"
usage_args="<command> <args>"
help_usage "$@"

2
sql

@ -1 +1 @@
Subproject commit c5add1631fa39dc79f129afb100146111a45d717
Subproject commit 77dd0422454591c2ec7163f83055eff9911eff69
Loading…
Cancel
Save