Merge branch 'master' of github.com:HariSekhon/DevOps-Bash-tools

pull/2/head
Hari Sekhon 3 years ago
commit 957b424f62

@ -20,7 +20,7 @@
srcdir="${srcdir:-$(dirname "${BASH_SOURCE[0]}")/..}"
# shellcheck disable=SC1090
# shellcheck disable=SC1090,SC1091
type add_PATH &>/dev/null || . "$srcdir/.bash.d/paths.sh"
# adds GCloud CLI tools to $PATH
@ -44,6 +44,7 @@ alias gke="gcloud container clusters"
alias gc="gcloud container"
alias gbs="gcloud builds submit --tag"
alias bqq="bq query"
alias gssh="gcloud compute ssh"
# when switching an alias to a function during re-source without un-aliasing, declare function explicitly to avoid errors
function gcloudconfig(){

@ -63,6 +63,7 @@ alias import=gitimport
alias co=checkout
alias commit="git commit"
alias clone="git clone"
alias cherry-pick="git cherry-pick"
alias gitci=commit
alias ci=commit
alias gitco=checkout

@ -54,6 +54,14 @@ fi
add_PATH "${KREW_ROOT:-$HOME/.krew}"
for x in "$bash_tools"/kubernetes*.sh; do
x="${x##*/}"
name="${x#kubernetes_}"
eval "k8s_${name}(){
'$x' \"\$@\"
}"
done
# ============================================================================ #
# replaced by function further down

@ -20,7 +20,8 @@
# https://cloud.google.com/sdk/gcloud/reference/topic/gcloudignore
# ============================================
# Ignore uploading these to Google App Engin, Cloud Build, Cloud Functions
# Ignore uploading these to Google Cloud Build, Cloud Functions, App Engine etc.
#
# there is a much bigger list in the adjacent .gitignore, some of which may be worth porting in to here too
#
@ -32,6 +33,7 @@
# since .* doesn't work, unlike .dockerignore we must specify the common dot files to exclude below
# .git seems to be ignored by default judging by cloud build uploaded tarball size
.git/
.hg/
.svn/

@ -137,6 +137,11 @@ Plugin 'tpope/vim-fugitive'
Plugin 'tpope/vim-surround'
"Plugin 'fatih/vim-go'
" comment at start of line instead of code indentation level
" doesn't work: https://github.com/preservim/nerdcommenter/issues/467
let g:NERDDefaultAlign = 'left'
let g:NERDCommentEmptyLines = 1
let g:gitgutter_enabled = 0
" keep setting if reloading, otherwise default to 1 for enabled
"let g:pluginname_setting = get(g:, 'gitgutter_enabled', 1)
@ -305,13 +310,15 @@ endif
nmap <silent> ;a :,!anonymize.py -a<CR>
nmap ;A :,!hexanonymize.py --case --hex-only<CR>
nmap <silent> ;b :!git blame "%"<CR>
nmap <silent> ;c :,!center.py<CR>
nmap <silent> ;c :call ToggleComments()<CR>
nmap <silent> ;C :,!center.py<CR>
nmap <silent> ;e :,!center.py -s<CR>
" parses current example line and passes as stdin to bash to quickly execute examples from code - see WriteRunLine() further down for example
" messes up interactive vim (disables vim's arrow keys) - calling a terminal reset fixes it
nmap <silent> ;E :call WriteRunLine()<CR> :!reset<CR><CR>
nmap <silent> ;d :r !date '+\%F \%T \%z (\%a, \%d \%b \%Y)'<CR>kJ
nmap <silent> ;D :Done<CR>
"nmap <silent> ;D :Done<CR>
nmap <silent> ;D :%!decomment.sh<CR>
nmap ;f :,!fold -s -w 120 \| sed 's/[[:space:]]*$//'<CR>
"nmap <silent> ;h :call Hr()<CR>
nmap <silent> ;h :Hr<CR>
@ -328,7 +335,9 @@ nmap ;n :n<CR>
nmap ;o :!git log -p "%"<CR>
nmap ;O :call ToggleGutter()<CR>
nmap ;p :prev<CR>
nmap ;P :call TogglePaste()<CR>
"nmap ;P :call TogglePaste()<CR>
nmap ;P :set paste!<CR>
nmap ;t :set list!<CR>
nmap ;q :q<CR>
nmap ;r :call WriteRun()<CR>
nmap ;R :call WriteRunDebug()<CR>
@ -393,15 +402,16 @@ function! ToggleSyntax()
endif
endfunction
"function! ToggleComment()
" let comment_prefix = '^' . b:comment_char
" echo comment_prefix
" if getline('.') =~ comment_prefix
" :s/^\=get(b:comment_char)//
" else
" :s/^/\=get(b:comment_char)/
" endif
"endfunction
function! ToggleComments()
:let comment_char = '#'
:let comment_prefix = '^' . comment_char
echo comment_prefix
if getline('.') =~ comment_prefix
:s/^\=:comment_char//
else
:s/^/\=:comment_char/
endif
endfunction
" setting this high keeps cursor in middle of screen
":set so=999
@ -413,13 +423,14 @@ function! ToggleScrollLock()
endif
endfunction
function! TogglePaste()
if &paste > 0
:set nopaste
else
:set paste
endif
endfunction
" simpler to call: set paste!
"function! TogglePaste()
" if &paste > 0
" :set nopaste
" else
" :set paste
" endif
"endfunction
" changing this setting has no effect on vim gutter in real time
function! ToggleGutter()

@ -317,6 +317,7 @@ etc.
- `aws_ecr_tag_image.sh` - tags an [AWS ECR](https://aws.amazon.com/ecr/) image with another tag without pulling and pushing it
- `aws_harden_password_policy.sh` - strengthens [AWS password policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_passwords_account-policy.html) according to [CIS Foundations Benchmark](https://d1.awsstatic.com/whitepapers/compliance/AWS_CIS_Foundations_Benchmark.pdf) recommendations
- `aws_iam_generate_credentials_report_wait.sh` - generates an AWS IAM [credentials report](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_getting-report.html)
- `aws_ip_ranges.sh` - get all AWS IP ranges for a given Region and/or Service using the IP range API
- `aws_kms_key_rotation_enabled.sh` - lists [AWS KMS](https://aws.amazon.com/kms/) keys and whether they have key rotation enabled
- `aws_kube_creds.sh` - auto-loads all AWS EKS clusters credentials in the current or given --region so your kubectl is ready to rock on AWS
- `aws_kubectl.sh` - runs kubectl commands safely fixed to a given AWS EKS cluster using config isolation to avoid concurrency race conditions
@ -849,7 +850,7 @@ etc.
### See Also:
* [DevOps Python Tools](https://github.com/harisekhon/devops-python-tools) - 80+ DevOps CLI tools for AWS, Hadoop, HBase, Spark, Log Anonymizer, Ambari Blueprints, AWS CloudFormation, Linux, Docker, Spark Data Converters & Validators (Avro / Parquet / JSON / CSV / INI / XML / YAML), Elasticsearch, Solr, Travis CI, Pig, IPython
* [DevOps Python Tools](https://github.com/harisekhon/devops-python-tools) - 80+ DevOps CLI tools for AWS, GCP, Hadoop, HBase, Spark, Log Anonymizer, Ambari Blueprints, AWS CloudFormation, Linux, Docker, Spark Data Converters & Validators (Avro / Parquet / JSON / CSV / INI / XML / YAML), Elasticsearch, Solr, Travis CI, Pig, IPython
* [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery

@ -0,0 +1,64 @@
#!/usr/bin/env bash
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2021-06-18 11:43:50 +0100 (Fri, 18 Jun 2021)
#
# https://github.com/HariSekhon/bash-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "${BASH_SOURCE[0]}")"
# shellcheck disable=SC1090,SC1091
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034,SC2154
usage_description="
Filter program to generate ArgoCD namespace resource whitelist from a give Kubernetes yaml or Kustomize build output
Yaml can be supplied as a file argument or via standard input. If no file is given, waits for stdin like a standard unix filter program
Outputs Yaml for the namespaceResourceWhitelist section of argocd-project.yaml
A full argocd-project.yaml is already provided at the URL below with all the most common object permissions already populated via the output from this script against my production environment
https://github.com/HariSekhon/Kubernetes-configs
Uses adjacent script kubernetes_resource_types.sh
Tested on ArgoCD 2.0.3
"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<file.yaml> <file2.yaml> ...]"
help_usage "$@"
#min_args 1 "$@"
echo " namespaceResourceWhitelist:"
"$srcdir/kubernetes_resource_types.sh" "$@" |
while read -r group kind; do
# Cluster resources, ignore these
if [[ "$kind" =~ Namespace|PriorityClass|StorageClass ]]; then
continue
fi
group="${group%/*}"
if [ "$group" = v1 ]; then
group=""
fi
if [ "$group" = "" ]; then
group="''"
fi
echo " - group: $group"
echo " kind: $kind"
done

@ -0,0 +1,109 @@
#!/usr/bin/env bash
# vim:ts=4:sts=4:sw=4:et
# args: eu-west-1 ROUTE53_HEALTHCHECKS
#
# Author: Hari Sekhon
# Date: 2021-06-18 17:13:10 +0100 (Fri, 18 Jun 2021)
#
# https://github.com/HariSekhon/bash-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "${BASH_SOURCE[0]}")"
# shellcheck disable=SC1090,SC1091
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034,SC2154
usage_description="
Returns all the AWS IPs for a given Region and Service using the AWS ip-range json API:
https://ip-ranges.amazonaws.com/ip-ranges.json
To get and use these IPs directly in Terraform, see the Cloudflare Firewall module in https://github.com/HariSekhon/Terraform
Examples:
Lists all regions and their services to filter on:
${0##*/} list
Get all IPs for eu-west-1 region:
${0##*/} eu-west-1
Get all eu-west-1 IPs for EC2, S3 or Route 53 Healthchecks:
${0##*/} eu-west-1 EC2
${0##*/} eu-west-1 S3
${0##*/} eu-west-1 ROUTE53_HEALTHCHECKS
Get global Route 53 Healthcheck IPs:
${0##*/} GLOBAL ROUTE53_HEALTHCHECKS
Get all Route 53 Healthcheck IPs in all regions:
${0##*/} all ROUTE53_HEALTHCHECKS
"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<region> <service> | list]"
help_usage "$@"
url="https://ip-ranges.amazonaws.com/ip-ranges.json"
region="${1:-}"
service="${2:-}"
# All regions are lowercase except for GLOBAL
region="$(tr '[:upper:]' '[:lower:]' <<< "$region")"
if [ "$region" = global ]; then
region=GLOBAL
fi
# All Services are uppercase
service="$(tr '[:lower:]' '[:upper:]' <<< "$service")"
if [ "$region" = list ]; then
curl -sS "$url" |
jq -r '.prefixes[] | [.region, .service] | @tsv' | sort -u
exit 0
fi
curl -sS "$url" |
#jq -r ".prefixes[]" |
#if [ -n "$region" ] && [ "$region" != all ]; then
# #jq -r ".prefixes[] | select(.region == \"$region\") | .ip_prefix"
# jq -r "select(.region == \"$region\")"
#else
# cat
#fi |
#if [ -n "$service" ] && [ "$service" != all ]; then
# jq -r "select(.service == \"$service\")"
#else
# cat
#fi |
#jq -r '.ip_prefix'
jq -r "
.prefixes[] |
if(\"$region\" != \"\" and \"$region\" != \"all\") then
select(.region == \"$region\")
else
.
end |
if(\"$service\" != \"\" and \"$service\" != \"all\") then
select(.service == \"$service\")
else
.
end |
.ip_prefix
" # end jq script

@ -31,6 +31,7 @@ For each commit in the current branch history, if both:
then both the Author and Committer names and emails are set to <new_name> and <new_email>
<git_options> - passed literally to git filter-branch after -- can use this to only rewrite a revision range, eg. <starting_hashref>..<ending_hashref>
Must be called from the top level directory of the repository
@ -47,7 +48,7 @@ If there is already a git filter-branch rewrite backup in .git/refs/original, gi
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="<old_email> <new_email> [<new_name>]"
usage_args="<old_email> <new_email> [<new_name>] [<git_options>]"
help_usage "$@"
@ -56,6 +57,11 @@ min_args 2 "$@"
old_email="$1"
new_email="$2"
new_name="${3:-}"
shift || :
shift || :
if [ -n "$new_name" ]; then
shift || :
fi
opts=()
if [ -n "${FORCE_GIT_REWRITE:-}" ]; then
@ -105,4 +111,4 @@ git filter-branch "${opts[@]}" --tag-name-filter cat --env-filter \
export GIT_COMMITTER_EMAIL='$new_email'
fi
" \
-- --all
-- --all "$@"

@ -43,7 +43,7 @@ help_usage "$@"
no_more_opts "$@"
awk '/namespace:/{print $2}' "$@" |
awk '/^[[:space:]]*namespace:[[:space:]]*[a-z0-9]([-a-z0-9]*[a-z0-9])?[[:space:]]*$/{print $2}' "$@" |
sort -u |
while read -r namespace; do
if kubectl get ns "$namespace" &>/dev/null; then

@ -0,0 +1,64 @@
#!/usr/bin/env bash
# vim:ts=4:sts=4:sw=4:et
#
# Author: Hari Sekhon
# Date: 2021-06-18 11:28:40 +0100 (Fri, 18 Jun 2021)
#
# https://github.com/HariSekhon/bash-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
#
# https://www.linkedin.com/in/HariSekhon
#
set -euo pipefail
[ -n "${DEBUG:-}" ] && set -x
srcdir="$(dirname "${BASH_SOURCE[0]}")"
# shellcheck disable=SC1090,SC1091
. "$srcdir/lib/utils.sh"
# shellcheck disable=SC2034,SC2154
usage_description="
Filter program to get all unique Kubernetes resources types out of a Kubernetes yaml or Kustomize build output
Yaml can be supplied as a file argument or via standard input. If no file is given, waits for stdin like a standard unix filter program
Output Format:
<group> <object_kind>
Sorted by object kind
eg.
v1 ConfigMap
batch/v1beta1 CronJob
apps/v1 Deployment
autoscaling/v1 HorizontalPodAutoscaler
extensions/v1beta1 Ingress
v1 Namespace
policy/v1beta1 PodDisruptionBudget
scheduling.k8s.io/v1 PriorityClass
v1 Service
v1 ServiceAccount
apps/v1 StatefulSet
storage.k8s.io/v1 StorageClass
autoscaling.k8s.io/v1beta2 VerticalPodAutoscaler
"
# used by usage() in lib/utils.sh
# shellcheck disable=SC2034
usage_args="[<file.yaml> <file2.yaml> ...]"
help_usage "$@"
#min_args 1 "$@"
awk '/^(api|kind)/{print $2}' "$@" |
# sed N joins every 2 lines
sed 'N;s/\n/ /' |
sort -k2 -u |
column -t

@ -40,6 +40,7 @@ amazon-ecs-cli
amazon-sam-cli
ansible
ansible-lint
argocd
#atomicparsley # mp4 cli tool
autoconf
autojump

@ -21,11 +21,33 @@ cd /tmp
# https://kubernetes-sigs.github.io/kustomize/installation/binaries/
date "+%F %T downloading kustomize"
curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
# now installs to /private and fails as user :-/
#curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash
# Kustomize 3.5.x gets an error like this when using kustomization.yaml with a double slash in the URL:
#
# Error: accumulating resources: accumulateFile "accumulating resources from 'github.com/argoproj/argo-cd//manifests/cluster-install?ref=v2.0.3': evalsymlink failure on '/tmp/git@repo/argocd/overlay/github.com/argoproj/argo-cd/manifests/cluster-install?ref=v2.0.3' : lstat /tmp/git@repo/argocd/overlay/github.com: no such file or directory", accumulateDirector: "recursed accumulation of path '/tmp/kustomize-881686007/repo': accumulating resources: accumulateFile \"accumulating resources from '../namespace-install': evalsymlink failure on '/tmp/kustomize-881686007/namespace-install' : lstat /tmp/kustomize-881686007/namespace-install: no such file or directory\", loader.New \"Error loading ../namespace-install with git: url lacks host: ../namespace-install, dir: evalsymlink failure on '/tmp/kustomize-881686007/namespace-install' : lstat /tmp/kustomize-881686007/namespace-install: no such file or directory, get: invalid source string: ../namespace-install\""
#
VERSION="${1:-4.1.3}"
os="$(uname -s | tr '[:upper:]' '[:lower:]')"
url="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv$VERSION/kustomize_v${VERSION}_${os}_amd64.tar.gz"
cd /tmp
wget "$url" -O kustomize.tar.gz
date "+%F %T unpacking kustomize"
tar zxvf kustomize.tar.gz
mkdir -pv ~/bin
unalias mv &>/dev/null || :
mv -vf kustomize ~/bin/
# called as part of download script
#~/bin/kustomize version -
rm -f kustomize.tar.gz
echo
# called as part of download script - call manually now
~/bin/kustomize version -

Loading…
Cancel
Save