Merge branch 'master' of github.com:HariSekhon/DevOps-Bash-tools
Hari Sekhon
5 years ago
commit
c6624d44de
@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
# vim:ts=4:sts=4:sw=4:et
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2019-12-10 11:33:52 +0000 (Tue, 10 Dec 2019)
|
||||
#
|
||||
# https://github.com/harisekhon/bash-tools
|
||||
#
|
||||
# License: see accompanying Hari Sekhon LICENSE file
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
|
||||
#
|
||||
# https://www.linkedin.com/in/harisekhon
|
||||
#
|
||||
|
||||
# Print each table's DDL metadata field eg. Location
|
||||
#
|
||||
# FILTER environment variable will restrict to matching fully qualified tables (<db>.<table>)
|
||||
#
|
||||
# Tested on Hive 1.1.0 on CDH 5.10, 5.16
|
||||
#
|
||||
# For more documentation see the comments at the top of beeline.sh
|
||||
|
||||
# For a better version written in Python see DevOps Python tools repo:
|
||||
#
|
||||
# https://github.com/harisekhon/devops-python-tools
|
||||
|
||||
# you will almost certainly have to comment out / remove '-o pipefail' to skip authorization errors such as that documented in impala_list_tables.sh
|
||||
set -eu # -o pipefail
|
||||
[ -n "${DEBUG:-}" ] && set -x
|
||||
srcdir="$(dirname "$0")"
|
||||
|
||||
usage(){
|
||||
echo "usage: ${0##*/} [beeline_options]"
|
||||
exit 3
|
||||
}
|
||||
|
||||
for arg; do
|
||||
if [[ "$arg" =~ -h|--help ]]; then
|
||||
usage
|
||||
fi
|
||||
done
|
||||
|
||||
query_template="describe {table}"
|
||||
|
||||
opts="--silent=true --outputformat=tsv2"
|
||||
|
||||
# exit the loop subshell if you Control-C
|
||||
trap 'exit 130' INT
|
||||
|
||||
"$srcdir/hive_list_tables.sh" "$@" |
|
||||
while read -r db table; do
|
||||
printf '%s.%s\t' "$db" "$table"
|
||||
query="${query_template//\{db\}/\`$db\`}"
|
||||
query="${query//\{table\}/\`$table\`}"
|
||||
# shellcheck disable=SC2086
|
||||
if ! "$srcdir/beeline.sh" $opts -e "USE \`$db\`; $query" "$@"; then
|
||||
echo "ERROR running query: $query" >&2
|
||||
echo "UNKNOWN"
|
||||
fi |
|
||||
tail -n +2 |
|
||||
awk '{if(NF == 2){print}}' |
|
||||
wc -l
|
||||
done
|
||||
@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# vim:ts=4:sts=4:sw=4:et
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2019-12-10 11:33:52 +0000 (Tue, 10 Dec 2019)
|
||||
#
|
||||
# https://github.com/harisekhon/bash-tools
|
||||
#
|
||||
# License: see accompanying Hari Sekhon LICENSE file
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
|
||||
#
|
||||
# https://www.linkedin.com/in/harisekhon
|
||||
#
|
||||
|
||||
# Print each table's number of columns
|
||||
#
|
||||
# FILTER environment variable will restrict to matching fully qualified tables (<db>.<table>)
|
||||
#
|
||||
# Caveats:
|
||||
#
|
||||
# Hive is more reliable as Impala breaks on some table metadata definitions where Hive doesn't
|
||||
#
|
||||
# Impala is faster than Hive for the first ~1000 tables but then slows down
|
||||
# so if you have a lot of tables I recommend you use the Hive version of this instead
|
||||
#
|
||||
# Tested on Impala 2.7.0, 2.12.0 on CDH 5.10, 5.16 with Kerberos and SSL
|
||||
#
|
||||
# For more documentation see the comments at the top of impala_shell.sh
|
||||
|
||||
# For a better version written in Python see DevOps Python tools repo:
|
||||
#
|
||||
# https://github.com/harisekhon/devops-python-tools
|
||||
|
||||
# you will almost certainly have to comment out / remove '-o pipefail' to skip authorization errors such as that documented in impala_list_tables.sh
|
||||
set -eu # -o pipefail
|
||||
[ -n "${DEBUG:-}" ] && set -x
|
||||
srcdir="$(dirname "$0")"
|
||||
|
||||
usage(){
|
||||
echo "usage: ${0##*/} [impala_shell_options]"
|
||||
exit 3
|
||||
}
|
||||
|
||||
for arg; do
|
||||
if [[ "$arg" =~ -h|--help ]]; then
|
||||
usage
|
||||
fi
|
||||
done
|
||||
|
||||
query_template="describe {table}"
|
||||
|
||||
# exit the loop subshell if you Control-C
|
||||
trap 'exit 130' INT
|
||||
|
||||
"$srcdir/impala_list_tables.sh" "$@" |
|
||||
while read -r db table; do
|
||||
printf '%s.%s\t' "$db" "$table"
|
||||
query="${query_template//\{db\}/\`$db\`}"
|
||||
query="${query//\{table\}/\`$table\`}"
|
||||
if ! "$srcdir/impala_shell.sh" --quiet -Bq "USE \`$db\`; $query" "$@"; then
|
||||
echo "ERROR running query: $query" >&2
|
||||
echo "UNKNOWN"
|
||||
fi |
|
||||
awk '{if(NF == 2){print}}' |
|
||||
wc -l
|
||||
done
|
||||
@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
# vim:ts=4:sts=4:sw=4:et
|
||||
#
|
||||
# Author: Hari Sekhon
|
||||
# Date: 2020-02-21 09:52:05 +0000 (Fri, 21 Feb 2020)
|
||||
# forked from pylib's Makefile from:
|
||||
# Original Date: 2013-01-06 15:45:00 +0000 (Sun, 06 Jan 2013)
|
||||
#
|
||||
# https://github.com/harisekhon/bash-tools
|
||||
#
|
||||
# License: see accompanying Hari Sekhon LICENSE file
|
||||
#
|
||||
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
|
||||
#
|
||||
# https://www.linkedin.com/in/harisekhon
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
[ -n "${DEBUG:-}" ] && set -x
|
||||
|
||||
sudo=""
|
||||
[ $EUID -eq 0 ] || sudo=sudo
|
||||
|
||||
#$sudo ln -sv `type -P python2` /usr/local/bin/python
|
||||
|
||||
set +e
|
||||
python2="$(type -P python2 2>/dev/null)"
|
||||
python3="$(type -P python3 2>/dev/null)"
|
||||
pip2="$(type -P pip2 2>/dev/null)"
|
||||
pip3="$(type -P pip3 2>/dev/null)"
|
||||
set -e
|
||||
|
||||
if ! type -P python; then
|
||||
if [ -n "$python3" ]; then
|
||||
$sudo alternatives --set python "$python3"
|
||||
elif [ -n "$python2" ]; then
|
||||
$sudo alternatives --set python "$python2"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! type -P pip; then
|
||||
if [ -f /usr/local/bin/pip ]; then
|
||||
echo "/usr/local/bin/pip already exists, not symlinking - check your \$PATH includes /usr/local/bin (\$PATH = $PATH)"
|
||||
elif [ -n "$pip3" ]; then
|
||||
$sudo ln -sv "$pip3" /usr/local/bin/pip
|
||||
elif [ -n "$pip2" ]; then
|
||||
$sudo ln -sv "$pip2" /usr/local/bin/pip
|
||||
else
|
||||
$sudo easy_install pip || :
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
python -V
|
||||
echo
|
||||
pip -V
|
||||
echo
|
||||
Loading…
Reference in New Issue