From 80c85a300d2ed378d4e5663feeba2e2b89d2a8a2 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Wed, 9 Jun 2021 11:21:38 +0100 Subject: [PATCH 01/35] updated brew-packages-desktop.txt --- setup/brew-packages-desktop.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/brew-packages-desktop.txt b/setup/brew-packages-desktop.txt index c24971f7..9d05b1e9 100644 --- a/setup/brew-packages-desktop.txt +++ b/setup/brew-packages-desktop.txt @@ -40,6 +40,7 @@ amazon-ecs-cli amazon-sam-cli ansible ansible-lint +argocd #atomicparsley # mp4 cli tool autoconf autojump From 9f093f434fc6f714b1e730e568579e82bbab1ec9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 10 Jun 2021 19:22:30 +0100 Subject: [PATCH 02/35] updated .vimrc --- .vimrc | 46 ++++++++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/.vimrc b/.vimrc index 1f65a9d7..0719fa9d 100644 --- a/.vimrc +++ b/.vimrc @@ -137,6 +137,11 @@ Plugin 'tpope/vim-fugitive' Plugin 'tpope/vim-surround' "Plugin 'fatih/vim-go' +" comment at start of line instead of code indentation level +" doesn't work: https://github.com/preservim/nerdcommenter/issues/467 +let g:NERDDefaultAlign = 'left' +let g:NERDCommentEmptyLines = 1 + let g:gitgutter_enabled = 0 " keep setting if reloading, otherwise default to 1 for enabled "let g:pluginname_setting = get(g:, 'gitgutter_enabled', 1) @@ -305,7 +310,8 @@ endif nmap ;a :,!anonymize.py -a nmap ;A :,!hexanonymize.py --case --hex-only nmap ;b :!git blame "%" -nmap ;c :,!center.py +nmap ;c :call ToggleComments() +nmap ;C :,!center.py nmap ;e :,!center.py -s " parses current example line and passes as stdin to bash to quickly execute examples from code - see WriteRunLine() further down for example " messes up interactive vim (disables vim's arrow keys) - calling a terminal reset fixes it @@ -328,7 +334,9 @@ nmap ;n :n nmap ;o :!git log -p "%" nmap ;O :call ToggleGutter() nmap ;p :prev -nmap ;P :call TogglePaste() +"nmap ;P :call TogglePaste() +nmap ;P :set paste! +nmap ;t :set list! nmap ;q :q nmap ;r :call WriteRun() nmap ;R :call WriteRunDebug() @@ -393,15 +401,16 @@ function! ToggleSyntax() endif endfunction -"function! ToggleComment() -" let comment_prefix = '^' . b:comment_char -" echo comment_prefix -" if getline('.') =~ comment_prefix -" :s/^\=get(b:comment_char)// -" else -" :s/^/\=get(b:comment_char)/ -" endif -"endfunction +function! ToggleComments() + :let comment_char = '#' + :let comment_prefix = '^' . comment_char + echo comment_prefix + if getline('.') =~ comment_prefix + :s/^\=:comment_char// + else + :s/^/\=:comment_char/ + endif +endfunction " setting this high keeps cursor in middle of screen ":set so=999 @@ -413,13 +422,14 @@ function! ToggleScrollLock() endif endfunction -function! TogglePaste() - if &paste > 0 - :set nopaste - else - :set paste - endif -endfunction +" simpler to call: set paste! +"function! TogglePaste() +" if &paste > 0 +" :set nopaste +" else +" :set paste +" endif +"endfunction " changing this setting has no effect on vim gutter in real time function! ToggleGutter() From 48c9b78d60846754c7afc0a80182a2ca947e567b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 11 Jun 2021 19:18:07 +0100 Subject: [PATCH 03/35] updated .vimrc --- .vimrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vimrc b/.vimrc index 0719fa9d..0abdcb59 100644 --- a/.vimrc +++ b/.vimrc @@ -317,7 +317,8 @@ nmap ;e :,!center.py -s " messes up interactive vim (disables vim's arrow keys) - calling a terminal reset fixes it nmap ;E :call WriteRunLine() :!reset nmap ;d :r !date '+\%F \%T \%z (\%a, \%d \%b \%Y)'kJ -nmap ;D :Done +"nmap ;D :Done +nmap ;D :%!decomment.sh nmap ;f :,!fold -s -w 120 \| sed 's/[[:space:]]*$//' "nmap ;h :call Hr() nmap ;h :Hr From 74d61584848399897dc7ae28e52ed2f209b02efa Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 14 Jun 2021 11:05:37 +0100 Subject: [PATCH 04/35] updated install_kustomize.sh --- setup/install_kustomize.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/setup/install_kustomize.sh b/setup/install_kustomize.sh index a7edd15e..c91bf790 100755 --- a/setup/install_kustomize.sh +++ b/setup/install_kustomize.sh @@ -21,7 +21,20 @@ cd /tmp # https://kubernetes-sigs.github.io/kustomize/installation/binaries/ date "+%F %T downloading kustomize" -curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash +# now installs to /private and fails as user :-/ +#curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash + +VERSION="4.1.3" + +os="$(uname -s | tr '[:upper:]' '[:lower:]')" + +url="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv$VERSION/kustomize_v${VERSION}_${os}_amd64.tar.gz" + +cd /tmp + +wget "$url" -O kustomize.tar.gz + +tar zxvf kustomize.tar.gz mkdir -pv ~/bin unalias mv &>/dev/null || : From fffbde88cb8f29fd7a0a9e4e36a23ac995ef6cd8 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 14 Jun 2021 11:07:06 +0100 Subject: [PATCH 05/35] updated install_kustomize.sh --- setup/install_kustomize.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/setup/install_kustomize.sh b/setup/install_kustomize.sh index c91bf790..d2de9eba 100755 --- a/setup/install_kustomize.sh +++ b/setup/install_kustomize.sh @@ -34,11 +34,16 @@ cd /tmp wget "$url" -O kustomize.tar.gz +date "+%F %T unpacking kustomize" tar zxvf kustomize.tar.gz mkdir -pv ~/bin unalias mv &>/dev/null || : mv -vf kustomize ~/bin/ -# called as part of download script -#~/bin/kustomize version - +rm -f kustomize.tar.gz + +echo + +# called as part of download script - call manually now +~/bin/kustomize version - From f8954dd8f7af2c9365119fb66f9c57c30eef1ea7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 14 Jun 2021 13:10:20 +0100 Subject: [PATCH 06/35] updated kubectl_create_namespaces.sh --- kubectl_create_namespaces.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kubectl_create_namespaces.sh b/kubectl_create_namespaces.sh index b737c737..a034998b 100755 --- a/kubectl_create_namespaces.sh +++ b/kubectl_create_namespaces.sh @@ -44,6 +44,7 @@ help_usage "$@" no_more_opts "$@" awk '/namespace:/{print $2}' "$@" | +sed '/^[[:space:]]*$/d' | sort -u | while read -r namespace; do if kubectl get ns "$namespace" &>/dev/null; then From 908a8135a7148c458c58a83b497e7a7d35b230b9 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Mon, 14 Jun 2021 13:23:56 +0100 Subject: [PATCH 07/35] updated install_kustomize.sh --- setup/install_kustomize.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/install_kustomize.sh b/setup/install_kustomize.sh index d2de9eba..d3b7490c 100755 --- a/setup/install_kustomize.sh +++ b/setup/install_kustomize.sh @@ -24,7 +24,7 @@ date "+%F %T downloading kustomize" # now installs to /private and fails as user :-/ #curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -VERSION="4.1.3" +VERSION="${1:-4.1.3}" os="$(uname -s | tr '[:upper:]' '[:lower:]')" From 87363f6963765da704b4b69846b4edfc6e63a84f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 15 Jun 2021 11:31:55 +0100 Subject: [PATCH 08/35] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e3608ce..6d86a4d3 100644 --- a/README.md +++ b/README.md @@ -849,7 +849,7 @@ etc. ### See Also: -* [DevOps Python Tools](https://github.com/harisekhon/devops-python-tools) - 80+ DevOps CLI tools for AWS, Hadoop, HBase, Spark, Log Anonymizer, Ambari Blueprints, AWS CloudFormation, Linux, Docker, Spark Data Converters & Validators (Avro / Parquet / JSON / CSV / INI / XML / YAML), Elasticsearch, Solr, Travis CI, Pig, IPython +* [DevOps Python Tools](https://github.com/harisekhon/devops-python-tools) - 80+ DevOps CLI tools for AWS, GCP, Hadoop, HBase, Spark, Log Anonymizer, Ambari Blueprints, AWS CloudFormation, Linux, Docker, Spark Data Converters & Validators (Avro / Parquet / JSON / CSV / INI / XML / YAML), Elasticsearch, Solr, Travis CI, Pig, IPython * [SQL Scripts](https://github.com/HariSekhon/SQL-scripts) - 100+ SQL Scripts - PostgreSQL, MySQL, AWS Athena, Google BigQuery From b95ca60d09fa95c4b7c906a6f5d1006cd0e1e8db Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Tue, 15 Jun 2021 13:00:16 +0100 Subject: [PATCH 09/35] updated kubectl_create_namespaces.sh --- kubectl_create_namespaces.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kubectl_create_namespaces.sh b/kubectl_create_namespaces.sh index a034998b..dc1a64af 100755 --- a/kubectl_create_namespaces.sh +++ b/kubectl_create_namespaces.sh @@ -43,8 +43,7 @@ help_usage "$@" no_more_opts "$@" -awk '/namespace:/{print $2}' "$@" | -sed '/^[[:space:]]*$/d' | +awk '/^[[:space:]]*namespace:[[:space:]]*[a-z0-9]([-a-z0-9]*[a-z0-9])?[[:space:]]*$/{print $2}' "$@" | sort -u | while read -r namespace; do if kubectl get ns "$namespace" &>/dev/null; then From dd52c18b7d97f09cfc318a062af5c0af6995aa80 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 11:09:10 +0100 Subject: [PATCH 10/35] updated .gcloudignore --- .gcloudignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gcloudignore b/.gcloudignore index 6ccf98b5..f40f3c81 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -20,7 +20,7 @@ # https://cloud.google.com/sdk/gcloud/reference/topic/gcloudignore # ============================================ -# Ignore uploading these to Google App Engin, Cloud Build, Cloud Functions +# Ignore uploading these to Google App Engine, Cloud Build, Cloud Functions # # there is a much bigger list in the adjacent .gitignore, some of which may be worth porting in to here too # From 05f3a84b72cea2309d1d4329e3d2e6aee67e0474 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 11:11:40 +0100 Subject: [PATCH 11/35] updated .gcloudignore --- .gcloudignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gcloudignore b/.gcloudignore index f40f3c81..804f4687 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -20,7 +20,8 @@ # https://cloud.google.com/sdk/gcloud/reference/topic/gcloudignore # ============================================ -# Ignore uploading these to Google App Engine, Cloud Build, Cloud Functions +# Ignore uploading these to Google Cloud Build, Cloud Functions, App Engine etc. + # # there is a much bigger list in the adjacent .gitignore, some of which may be worth porting in to here too # @@ -32,6 +33,7 @@ # since .* doesn't work, unlike .dockerignore we must specify the common dot files to exclude below +# .git seems to be ignored by default judging by cloud build uploaded tarball size .git/ .hg/ .svn/ From 07a17e872e580f98f70b7bc37fe2d43f069e93ab Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 11:14:37 +0100 Subject: [PATCH 12/35] updated git.sh --- .bash.d/git.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.bash.d/git.sh b/.bash.d/git.sh index 1680397f..bd9fa72e 100644 --- a/.bash.d/git.sh +++ b/.bash.d/git.sh @@ -63,6 +63,7 @@ alias import=gitimport alias co=checkout alias commit="git commit" alias clone="git clone" +alias cherry-pick="git cherry-pick" alias gitci=commit alias ci=commit alias gitco=checkout From aa79e3931547ce74788f4ec7b910c0ecbb9edebd Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 11:32:12 +0100 Subject: [PATCH 13/35] added kubernetes_resource_types.sh --- kubernetes_resource_types.sh | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 kubernetes_resource_types.sh diff --git a/kubernetes_resource_types.sh b/kubernetes_resource_types.sh new file mode 100755 index 00000000..2b3c3f52 --- /dev/null +++ b/kubernetes_resource_types.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-06-18 11:28:40 +0100 (Fri, 18 Jun 2021) +# +# https://github.com/HariSekhon/bash-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(dirname "${BASH_SOURCE[0]}")" + +# shellcheck disable=SC1090,SC1091 +. "$srcdir/lib/utils.sh" + +# shellcheck disable=SC2034,SC2154 +usage_description=" +Filter program to get all unique Kubernetes resources types out of a Kubernetes yaml or Kustomize build output + +Yaml can be supplied as a file argument or via standard input. If not file is given, waits for stdin like a standard unix filter program. +" + +# used by usage() in lib/utils.sh +# shellcheck disable=SC2034 +usage_args="[]" + +help_usage "$@" + +#min_args 1 "$@" + +awk '/^(api|kind)/{print $2}' "$@" | +sed 'N;s/\n/ /' | +column -t | +sort -u From 3a0dd4d7e08070bdab9f27e905d8a165881e5704 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 11:38:17 +0100 Subject: [PATCH 14/35] updated kubernetes.sh --- .bash.d/kubernetes.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.bash.d/kubernetes.sh b/.bash.d/kubernetes.sh index 15b4b91b..28792014 100644 --- a/.bash.d/kubernetes.sh +++ b/.bash.d/kubernetes.sh @@ -54,6 +54,14 @@ fi add_PATH "${KREW_ROOT:-$HOME/.krew}" +for x in "$bash_tools"/kubernetes*.sh; do + x="${x##*/}" + name="${x#kubernetes_}" + eval "k8s_${name}(){ + '$x' \"\$@\" + }" +done + # ============================================================================ # # replaced by function further down From 1ec3a99ad8ad4812c1e0d932a9568055ba642b89 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 11:41:59 +0100 Subject: [PATCH 15/35] updated kubernetes_resource_types.sh --- kubernetes_resource_types.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes_resource_types.sh b/kubernetes_resource_types.sh index 2b3c3f52..71914dab 100755 --- a/kubernetes_resource_types.sh +++ b/kubernetes_resource_types.sh @@ -36,6 +36,7 @@ help_usage "$@" #min_args 1 "$@" awk '/^(api|kind)/{print $2}' "$@" | +# sed N joins every 2 lines sed 'N;s/\n/ /' | column -t | sort -u From b9efb19e2b2917c64067a531ab837263d959796b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 11:55:27 +0100 Subject: [PATCH 16/35] updated kubernetes_resource_types.sh --- kubernetes_resource_types.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes_resource_types.sh b/kubernetes_resource_types.sh index 71914dab..8b571c40 100755 --- a/kubernetes_resource_types.sh +++ b/kubernetes_resource_types.sh @@ -24,12 +24,12 @@ srcdir="$(dirname "${BASH_SOURCE[0]}")" usage_description=" Filter program to get all unique Kubernetes resources types out of a Kubernetes yaml or Kustomize build output -Yaml can be supplied as a file argument or via standard input. If not file is given, waits for stdin like a standard unix filter program. +Yaml can be supplied as a file argument or via standard input. If no file is given, waits for stdin like a standard unix filter program " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 -usage_args="[]" +usage_args="[ ...]" help_usage "$@" From acfdb175fd619363129a50b9f170bae01eac641a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 12:06:43 +0100 Subject: [PATCH 17/35] added argocd_namespace_resource_whitelist.sh --- argocd_namespace_resource_whitelist.sh | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100755 argocd_namespace_resource_whitelist.sh diff --git a/argocd_namespace_resource_whitelist.sh b/argocd_namespace_resource_whitelist.sh new file mode 100755 index 00000000..691210f4 --- /dev/null +++ b/argocd_namespace_resource_whitelist.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2021-06-18 11:43:50 +0100 (Fri, 18 Jun 2021) +# +# https://github.com/HariSekhon/bash-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(dirname "${BASH_SOURCE[0]}")" + +# shellcheck disable=SC1090,SC1091 +. "$srcdir/lib/utils.sh" + +# shellcheck disable=SC2034,SC2154 +usage_description=" +Filter program to generate ArgoCD namespace resource whitelist from a give Kubernetes yaml or Kustomize build output + +Yaml can be supplied as a file argument or via standard input. If no file is given, waits for stdin like a standard unix filter program + +Outputs Yaml for the namespaceResourceWhitelist section of argocd-project.yaml + +Uses adjacent script kubernetes_resource_types.sh + +Tested on ArgoCD 2.0.3 +" + +# used by usage() in lib/utils.sh +# shellcheck disable=SC2034 +usage_args="[ ...]" + +help_usage "$@" + +#min_args 1 "$@" + +echo " namespaceResourceWhitelist:" +"$srcdir/kubernetes_resource_types.sh" "$@" | +while read -r group kind; do + if [[ "$kind" =~ Namespace|PriorityClass|StorageClass ]]; then + continue + fi + group="${group%/*}" + if [ "$group" = v1 ]; then + group="" + fi + echo " - group: '$group'" + echo " kind: '$kind'" +done From f1df2145e024a1f2b6e7056413a064f490fb341b Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 12:20:30 +0100 Subject: [PATCH 18/35] updated argocd_namespace_resource_whitelist.sh --- argocd_namespace_resource_whitelist.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/argocd_namespace_resource_whitelist.sh b/argocd_namespace_resource_whitelist.sh index 691210f4..11d296fc 100755 --- a/argocd_namespace_resource_whitelist.sh +++ b/argocd_namespace_resource_whitelist.sh @@ -28,6 +28,10 @@ Yaml can be supplied as a file argument or via standard input. If no file is giv Outputs Yaml for the namespaceResourceWhitelist section of argocd-project.yaml +A full argocd-project.yaml is already provided at the URL below with all the most common object permissions already populated via the output from this script against my production environment + + https://github.com/HariSekhon/Kubernetes-configs + Uses adjacent script kubernetes_resource_types.sh Tested on ArgoCD 2.0.3 @@ -44,6 +48,7 @@ help_usage "$@" echo " namespaceResourceWhitelist:" "$srcdir/kubernetes_resource_types.sh" "$@" | while read -r group kind; do + # Cluster resources, ignore these if [[ "$kind" =~ Namespace|PriorityClass|StorageClass ]]; then continue fi From 65de9c89a743ba090056e9d6b376efbb57cfb70f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 12:23:57 +0100 Subject: [PATCH 19/35] updated kubernetes_resource_types.sh --- kubernetes_resource_types.sh | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/kubernetes_resource_types.sh b/kubernetes_resource_types.sh index 8b571c40..203b0c35 100755 --- a/kubernetes_resource_types.sh +++ b/kubernetes_resource_types.sh @@ -25,6 +25,28 @@ usage_description=" Filter program to get all unique Kubernetes resources types out of a Kubernetes yaml or Kustomize build output Yaml can be supplied as a file argument or via standard input. If no file is given, waits for stdin like a standard unix filter program + +Output Format: + + + +Sorted by object kind + +eg. + +v1 ConfigMap +batch/v1beta1 CronJob +apps/v1 Deployment +autoscaling/v1 HorizontalPodAutoscaler +extensions/v1beta1 Ingress +v1 Namespace +policy/v1beta1 PodDisruptionBudget +scheduling.k8s.io/v1 PriorityClass +v1 Service +v1 ServiceAccount +apps/v1 StatefulSet +storage.k8s.io/v1 StorageClass +autoscaling.k8s.io/v1beta2 VerticalPodAutoscaler " # used by usage() in lib/utils.sh @@ -38,5 +60,5 @@ help_usage "$@" awk '/^(api|kind)/{print $2}' "$@" | # sed N joins every 2 lines sed 'N;s/\n/ /' | -column -t | -sort -u +sort -k2 -u | +column -t From 81afd37ef0e9f101336f715db0d068400a3d66f7 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 12:26:35 +0100 Subject: [PATCH 20/35] updated argocd_namespace_resource_whitelist.sh --- argocd_namespace_resource_whitelist.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/argocd_namespace_resource_whitelist.sh b/argocd_namespace_resource_whitelist.sh index 11d296fc..0979a16d 100755 --- a/argocd_namespace_resource_whitelist.sh +++ b/argocd_namespace_resource_whitelist.sh @@ -56,6 +56,9 @@ while read -r group kind; do if [ "$group" = v1 ]; then group="" fi - echo " - group: '$group'" - echo " kind: '$kind'" + if [ "$group" = "" ]; then + group="''" + fi + echo " - group: $group" + echo " kind: $kind" done From 8e3f162bb4f45ef1748fa6fc2fd4ecdf7452a47f Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:29:06 +0100 Subject: [PATCH 21/35] added aws_ips.sh --- aws_ips.sh | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 aws_ips.sh diff --git a/aws_ips.sh b/aws_ips.sh new file mode 100755 index 00000000..a3c22a1c --- /dev/null +++ b/aws_ips.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# args: eu-west-1 ROUTE53_HEALTHCHECKS +# +# Author: Hari Sekhon +# Date: 2021-06-18 17:13:10 +0100 (Fri, 18 Jun 2021) +# +# https://github.com/HariSekhon/bash-tools +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +srcdir="$(dirname "${BASH_SOURCE[0]}")" + +# shellcheck disable=SC1090,SC1091 +. "$srcdir/lib/utils.sh" + +# shellcheck disable=SC2034,SC2154 +usage_description=" +Returns all the AWS IPs for a given Region and Service using the AWS ip-range json API: + + https://ip-ranges.amazonaws.com/ip-ranges.json + +Examples: + + Lists all regions and their services to filter on: + + ${0##*/} list + + Get all IPs for eu-west-1 region: + + ${0##*/} eu-west-1 + + Get all eu-west-1 IPs for EC2, S3 or Route 53 Healthchecks: + + ${0##*/} eu-west-1 EC2 + ${0##*/} eu-west-1 S3 + ${0##*/} eu-west-1 ROUTE53_HEALTHCHECKS + + Get global Route 53 Healthcheck IPs: + + ${0##*/} GLOBAL ROUTE53_HEALTHCHECKS +" + +# used by usage() in lib/utils.sh +# shellcheck disable=SC2034 +usage_args="[ ]" + +help_usage "$@" + +#min_args 1 "$@" + +url="https://ip-ranges.amazonaws.com/ip-ranges.json" +region="${1:-}" +service="${2:-}" + +curl -sS "$url" | +jq -r ".prefixes[]" | +if [ "$region" == list ]; then + jq -r '[.region, .service] | @tsv' | sort -u >&2 +elif [ -n "$region" ]; then + #jq -r ".prefixes[] | select(.region == \"$region\") | .ip_prefix" + jq -r "select(.region == \"$region\")" +else + cat +fi | +if [ -n "$service" ]; then + jq -r "select(.service == \"$service\")" +else + cat +fi | +jq -r '.ip_prefix' From 7a9a5b37f7f5b6b71e5008cd08315568c9b5cabc Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:29:58 +0100 Subject: [PATCH 22/35] updated README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6d86a4d3..49a933f3 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,7 @@ etc. - `aws_ecr_tag_image.sh` - tags an [AWS ECR](https://aws.amazon.com/ecr/) image with another tag without pulling and pushing it - `aws_harden_password_policy.sh` - strengthens [AWS password policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_passwords_account-policy.html) according to [CIS Foundations Benchmark](https://d1.awsstatic.com/whitepapers/compliance/AWS_CIS_Foundations_Benchmark.pdf) recommendations - `aws_iam_generate_credentials_report_wait.sh` - generates an AWS IAM [credentials report](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_getting-report.html) + - `aws_ips.sh` - get all AWS IP ranges for a given Region and/or Service using the IP range json API - `aws_kms_key_rotation_enabled.sh` - lists [AWS KMS](https://aws.amazon.com/kms/) keys and whether they have key rotation enabled - `aws_kube_creds.sh` - auto-loads all AWS EKS clusters credentials in the current or given --region so your kubectl is ready to rock on AWS - `aws_kubectl.sh` - runs kubectl commands safely fixed to a given AWS EKS cluster using config isolation to avoid concurrency race conditions From 18b1db34b33aee46f690a9ce63a1faeeb99124d3 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:31:44 +0100 Subject: [PATCH 23/35] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49a933f3..417e5099 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,7 @@ etc. - `aws_ecr_tag_image.sh` - tags an [AWS ECR](https://aws.amazon.com/ecr/) image with another tag without pulling and pushing it - `aws_harden_password_policy.sh` - strengthens [AWS password policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_passwords_account-policy.html) according to [CIS Foundations Benchmark](https://d1.awsstatic.com/whitepapers/compliance/AWS_CIS_Foundations_Benchmark.pdf) recommendations - `aws_iam_generate_credentials_report_wait.sh` - generates an AWS IAM [credentials report](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_getting-report.html) - - `aws_ips.sh` - get all AWS IP ranges for a given Region and/or Service using the IP range json API + - `aws_ip_ranges.sh` - get all AWS IP ranges for a given Region and/or Service using the IP range json API - `aws_kms_key_rotation_enabled.sh` - lists [AWS KMS](https://aws.amazon.com/kms/) keys and whether they have key rotation enabled - `aws_kube_creds.sh` - auto-loads all AWS EKS clusters credentials in the current or given --region so your kubectl is ready to rock on AWS - `aws_kubectl.sh` - runs kubectl commands safely fixed to a given AWS EKS cluster using config isolation to avoid concurrency race conditions From 1586366e8981ff5aa83cc5d5cd122707ffd1f522 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:31:53 +0100 Subject: [PATCH 24/35] renamed aws_ips.sh to aws_ip_ranges.sh --- aws_ips.sh => aws_ip_ranges.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename aws_ips.sh => aws_ip_ranges.sh (100%) diff --git a/aws_ips.sh b/aws_ip_ranges.sh similarity index 100% rename from aws_ips.sh rename to aws_ip_ranges.sh From 8127b16039ae3104453a79baba34645029860544 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:38:16 +0100 Subject: [PATCH 25/35] updated aws_ip_ranges.sh --- aws_ip_ranges.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/aws_ip_ranges.sh b/aws_ip_ranges.sh index a3c22a1c..5f6b0616 100755 --- a/aws_ip_ranges.sh +++ b/aws_ip_ranges.sh @@ -60,11 +60,15 @@ url="https://ip-ranges.amazonaws.com/ip-ranges.json" region="${1:-}" service="${2:-}" +if [ "$region" = list ]; then + curl -sS "$url" | + jq -r '.prefixes[] | [.region, .service] | @tsv' | sort -u + exit 0 +fi + curl -sS "$url" | jq -r ".prefixes[]" | -if [ "$region" == list ]; then - jq -r '[.region, .service] | @tsv' | sort -u >&2 -elif [ -n "$region" ]; then +if [ -n "$region" ]; then #jq -r ".prefixes[] | select(.region == \"$region\") | .ip_prefix" jq -r "select(.region == \"$region\")" else From 3e93068a6f6c1cdc50f6c50982d83ea17225348e Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:41:28 +0100 Subject: [PATCH 26/35] updated aws_ip_ranges.sh --- aws_ip_ranges.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/aws_ip_ranges.sh b/aws_ip_ranges.sh index 5f6b0616..d4582f5e 100755 --- a/aws_ip_ranges.sh +++ b/aws_ip_ranges.sh @@ -46,11 +46,16 @@ Examples: Get global Route 53 Healthcheck IPs: ${0##*/} GLOBAL ROUTE53_HEALTHCHECKS + + Get all Route 53 Healthcheck IPs in all regions: + + ${0##*/} all ROUTE53_HEALTHCHECKS + " # used by usage() in lib/utils.sh # shellcheck disable=SC2034 -usage_args="[ ]" +usage_args="[ | list]" help_usage "$@" @@ -68,7 +73,7 @@ fi curl -sS "$url" | jq -r ".prefixes[]" | -if [ -n "$region" ]; then +if [ -n "$region" ] && [ "$region" != all ]; then #jq -r ".prefixes[] | select(.region == \"$region\") | .ip_prefix" jq -r "select(.region == \"$region\")" else From 87947179e5215b78dee3dffaf9fef363f0f88857 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:41:47 +0100 Subject: [PATCH 27/35] updated aws_ip_ranges.sh --- aws_ip_ranges.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_ip_ranges.sh b/aws_ip_ranges.sh index d4582f5e..d7c06b01 100755 --- a/aws_ip_ranges.sh +++ b/aws_ip_ranges.sh @@ -79,7 +79,7 @@ if [ -n "$region" ] && [ "$region" != all ]; then else cat fi | -if [ -n "$service" ]; then +if [ -n "$service" ] && [ "$service" != all ]; then jq -r "select(.service == \"$service\")" else cat From 99adfdbd4268dbb8c9060fec2bc019565d869606 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:48:39 +0100 Subject: [PATCH 28/35] updated aws_ip_ranges.sh --- aws_ip_ranges.sh | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/aws_ip_ranges.sh b/aws_ip_ranges.sh index d7c06b01..9b1bb776 100755 --- a/aws_ip_ranges.sh +++ b/aws_ip_ranges.sh @@ -72,16 +72,30 @@ if [ "$region" = list ]; then fi curl -sS "$url" | -jq -r ".prefixes[]" | -if [ -n "$region" ] && [ "$region" != all ]; then - #jq -r ".prefixes[] | select(.region == \"$region\") | .ip_prefix" - jq -r "select(.region == \"$region\")" -else - cat -fi | -if [ -n "$service" ] && [ "$service" != all ]; then - jq -r "select(.service == \"$service\")" -else - cat -fi | -jq -r '.ip_prefix' +#jq -r ".prefixes[]" | +#if [ -n "$region" ] && [ "$region" != all ]; then +# #jq -r ".prefixes[] | select(.region == \"$region\") | .ip_prefix" +# jq -r "select(.region == \"$region\")" +#else +# cat +#fi | +#if [ -n "$service" ] && [ "$service" != all ]; then +# jq -r "select(.service == \"$service\")" +#else +# cat +#fi | +#jq -r '.ip_prefix' +jq -r " + .prefixes[] | + if(\"$region\" != \"\" and \"$region\" != \"all\") then + select(.region == \"$region\") + else + . + end | + if(\"$service\" != \"\" and \"$service\" != \"all\") then + select(.service == \"$service\") + else + . + end | + .ip_prefix +" # end jq script From 37f430d784a7c131e5d0fd5be6eae869341f0117 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:50:15 +0100 Subject: [PATCH 29/35] updated README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 417e5099..38d04d87 100644 --- a/README.md +++ b/README.md @@ -317,7 +317,7 @@ etc. - `aws_ecr_tag_image.sh` - tags an [AWS ECR](https://aws.amazon.com/ecr/) image with another tag without pulling and pushing it - `aws_harden_password_policy.sh` - strengthens [AWS password policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_passwords_account-policy.html) according to [CIS Foundations Benchmark](https://d1.awsstatic.com/whitepapers/compliance/AWS_CIS_Foundations_Benchmark.pdf) recommendations - `aws_iam_generate_credentials_report_wait.sh` - generates an AWS IAM [credentials report](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_getting-report.html) - - `aws_ip_ranges.sh` - get all AWS IP ranges for a given Region and/or Service using the IP range json API + - `aws_ip_ranges.sh` - get all AWS IP ranges for a given Region and/or Service using the IP range API - `aws_kms_key_rotation_enabled.sh` - lists [AWS KMS](https://aws.amazon.com/kms/) keys and whether they have key rotation enabled - `aws_kube_creds.sh` - auto-loads all AWS EKS clusters credentials in the current or given --region so your kubectl is ready to rock on AWS - `aws_kubectl.sh` - runs kubectl commands safely fixed to a given AWS EKS cluster using config isolation to avoid concurrency race conditions From 13049e64a90b8984f8b536ae8100031e434cd074 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 17:54:04 +0100 Subject: [PATCH 30/35] updated aws_ip_ranges.sh --- aws_ip_ranges.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/aws_ip_ranges.sh b/aws_ip_ranges.sh index 9b1bb776..dc22355b 100755 --- a/aws_ip_ranges.sh +++ b/aws_ip_ranges.sh @@ -59,12 +59,18 @@ usage_args="[ | list]" help_usage "$@" -#min_args 1 "$@" - url="https://ip-ranges.amazonaws.com/ip-ranges.json" region="${1:-}" service="${2:-}" +# All regions are lowercase except for GLOBAL +region="$(tr '[:upper:]' '[:lower:]' <<< "$region")" +if [ "$region" = global ]; then + region=GLOBAL +fi +# All Services are uppercase +service="$(tr '[:lower:]' '[:upper:]' <<< "$service")" + if [ "$region" = list ]; then curl -sS "$url" | jq -r '.prefixes[] | [.region, .service] | @tsv' | sort -u From bb8d4d5b21b8b4d42a536221d82e4a7348150216 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 18:57:29 +0100 Subject: [PATCH 31/35] updated gcp.sh --- .bash.d/gcp.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.bash.d/gcp.sh b/.bash.d/gcp.sh index 84d7dc0e..e9bce91d 100644 --- a/.bash.d/gcp.sh +++ b/.bash.d/gcp.sh @@ -20,7 +20,7 @@ srcdir="${srcdir:-$(dirname "${BASH_SOURCE[0]}")/..}" -# shellcheck disable=SC1090 +# shellcheck disable=SC1090,SC1091 type add_PATH &>/dev/null || . "$srcdir/.bash.d/paths.sh" # adds GCloud CLI tools to $PATH @@ -44,6 +44,7 @@ alias gke="gcloud container clusters" alias gc="gcloud container" alias gbs="gcloud builds submit --tag" alias bqq="bq query" +alias gssh="gcloud compute ssh" # when switching an alias to a function during re-source without un-aliasing, declare function explicitly to avoid errors function gcloudconfig(){ From 16ca3dd4db527cb1f9cf4798a234ffe94876ca35 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 19:10:11 +0100 Subject: [PATCH 32/35] updated aws_ip_ranges.sh --- aws_ip_ranges.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws_ip_ranges.sh b/aws_ip_ranges.sh index dc22355b..b5c0e2d6 100755 --- a/aws_ip_ranges.sh +++ b/aws_ip_ranges.sh @@ -27,6 +27,8 @@ Returns all the AWS IPs for a given Region and Service using the AWS ip-range js https://ip-ranges.amazonaws.com/ip-ranges.json +To get and use these IPs directly in Terraform, see https://github.com/HariSekhon/Terraform + Examples: Lists all regions and their services to filter on: From da2099b9c9f7bd02cd210ec4628d8178ba1b358a Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 18 Jun 2021 19:10:33 +0100 Subject: [PATCH 33/35] updated aws_ip_ranges.sh --- aws_ip_ranges.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_ip_ranges.sh b/aws_ip_ranges.sh index b5c0e2d6..7a200298 100755 --- a/aws_ip_ranges.sh +++ b/aws_ip_ranges.sh @@ -27,7 +27,7 @@ Returns all the AWS IPs for a given Region and Service using the AWS ip-range js https://ip-ranges.amazonaws.com/ip-ranges.json -To get and use these IPs directly in Terraform, see https://github.com/HariSekhon/Terraform +To get and use these IPs directly in Terraform, see the Cloudflare Firewall module in https://github.com/HariSekhon/Terraform Examples: From 5ec076e032a98d44852c89edacad5e45df56c745 Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Thu, 24 Jun 2021 10:37:55 +0100 Subject: [PATCH 34/35] updated install_kustomize.sh --- setup/install_kustomize.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup/install_kustomize.sh b/setup/install_kustomize.sh index d3b7490c..07296968 100755 --- a/setup/install_kustomize.sh +++ b/setup/install_kustomize.sh @@ -24,6 +24,10 @@ date "+%F %T downloading kustomize" # now installs to /private and fails as user :-/ #curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash +# Kustomize 3.5.x gets an error like this when using kustomization.yaml with a double slash in the URL: +# +# Error: accumulating resources: accumulateFile "accumulating resources from 'github.com/argoproj/argo-cd//manifests/cluster-install?ref=v2.0.3': evalsymlink failure on '/tmp/git@repo/argocd/overlay/github.com/argoproj/argo-cd/manifests/cluster-install?ref=v2.0.3' : lstat /tmp/git@repo/argocd/overlay/github.com: no such file or directory", accumulateDirector: "recursed accumulation of path '/tmp/kustomize-881686007/repo': accumulating resources: accumulateFile \"accumulating resources from '../namespace-install': evalsymlink failure on '/tmp/kustomize-881686007/namespace-install' : lstat /tmp/kustomize-881686007/namespace-install: no such file or directory\", loader.New \"Error loading ../namespace-install with git: url lacks host: ../namespace-install, dir: evalsymlink failure on '/tmp/kustomize-881686007/namespace-install' : lstat /tmp/kustomize-881686007/namespace-install: no such file or directory, get: invalid source string: ../namespace-install\"" +# VERSION="${1:-4.1.3}" os="$(uname -s | tr '[:upper:]' '[:lower:]')" From 4e584064e1e654c27d7e21b28ab3c2e445a7243d Mon Sep 17 00:00:00 2001 From: Hari Sekhon Date: Fri, 25 Jun 2021 09:05:14 +0100 Subject: [PATCH 35/35] updated git_filter_branch_fix_author.sh --- git_filter_branch_fix_author.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/git_filter_branch_fix_author.sh b/git_filter_branch_fix_author.sh index d3b1358a..e2956812 100755 --- a/git_filter_branch_fix_author.sh +++ b/git_filter_branch_fix_author.sh @@ -31,6 +31,7 @@ For each commit in the current branch history, if both: then both the Author and Committer names and emails are set to and + - passed literally to git filter-branch after -- can use this to only rewrite a revision range, eg. .. Must be called from the top level directory of the repository @@ -47,7 +48,7 @@ If there is already a git filter-branch rewrite backup in .git/refs/original, gi # used by usage() in lib/utils.sh # shellcheck disable=SC2034 -usage_args=" []" +usage_args=" [] []" help_usage "$@" @@ -56,6 +57,11 @@ min_args 2 "$@" old_email="$1" new_email="$2" new_name="${3:-}" +shift || : +shift || : +if [ -n "$new_name" ]; then + shift || : +fi opts=() if [ -n "${FORCE_GIT_REWRITE:-}" ]; then @@ -105,4 +111,4 @@ git filter-branch "${opts[@]}" --tag-name-filter cat --env-filter \ export GIT_COMMITTER_EMAIL='$new_email' fi " \ - -- --all + -- --all "$@"