You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
DevOps-Bash-tools/Makefile.in

665 lines
20 KiB
Makefile

#
# Author: Hari Sekhon
# Date: 2013-02-03 10:25:36 +0000 (Sun, 03 Feb 2013)
#
# https://github.com/harisekhon/bash-tools
#
# License: see accompanying Hari Sekhon LICENSE file
#
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback
# to help improve or steer this or other code I publish
#
# https://www.linkedin.com/in/harisekhon
#
ifneq ("$(wildcard bash-tools)", "")
BASH_TOOLS := bash-tools
else
BASH_TOOLS := .
endif
export PATH := $(PATH):/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin
DOCKER_IMAGE := harisekhon/centos-github
CODE_FILES := $(shell git ls-files | while read line; do test -f "$$line" || continue; echo "$$line"; done)
CPANM = cpanm
FATPACKS_DIR := fatpacks
SUDO := sudo
SUDO_PIP := sudo -H
SUDO_PERL := sudo
PYTHON_VIRTUALENV :=
ifdef PERLBREW_PERL
# can't put this here, nor @commented, otherwise gets error - "commands commence before first target. Stop."
#echo "Perlbrew environment detected, not calling sudo"
SUDO_PERL =
endif
# Travis has custom python install earlier in $PATH even in Perl builds so need to install PyPI modules locally to non-system python otherwise they're not found by programs.
# Perms not set correctly on custom python install in Travis perl build so workaround is done to chown to travis user in .travis.yml
# Better than modifying $PATH to put /usr/bin first which is likely to affect many other things including potentially not finding the perlbrew installation first
# Looks like Perl travis builds are now using system Python - do not use TRAVIS env
ifdef VIRTUAL_ENV
#echo "Virtual Env / Conda detected, not calling sudo"
SUDO_PIP :=
PYTHON_VIRTUALENV := 1
endif
ifdef CONDA_DEFAULT_ENV
SUDO_PIP :=
PYTHON_VIRTUALENV := 1
endif
# must come after to reset SUDO_PERL/SUDO_PIP to blank if root
# EUID / UID not exported in Make
# USER not populated in Docker
ifeq '$(shell id -u)' '0'
#echo "root UID detected, not calling sudo"
SUDO :=
SUDO_PERL :=
SUDO_PIP :=
endif
# placeholders to silence check_makefile.sh warnings - should be set in client Makefiles after sourcing
ifndef REPO
REPO := NOTSET
endif
ifndef ARGS
ARGS := NOTSET
endif
ifndef CONF_FILES
CONF_FILES := NOTSET
endif
define MAKEFILE_USAGE_COMMON
Usage:
Common Options:
make help show this message
make build installs all dependencies - OS packages and any language libraries via native tools eg. pip, cpanm, gem, go etc that are not available via OS packages
make system-packages installs OS packages only (detects OS via whichever package manager is available)
make test run tests
make clean removes compiled / generated files, downloaded tarballs, temporary files etc.
make submodules initialize and update submodules to the right release (done automatically by build / system-packages)
make cpan install any modules listed in any cpan-requirements.txt files if not already installed
make pip install any modules listed in any requirements.txt files if not already installed
make python-compile compile any python files found in the current directory and 1 level of subdirectory
make pycompile
make github open browser at github project
make readme open browser at github's README
make github-url print github url and copy to clipboard
make ls-files print list of files in project
make ls-code print list of code files, excluding READMEs and other peripheral files
make wc show line counts of the files and grand total
make wc-code show line counts of only code files and total
endef
#make ${VENV} make a virtualenv in the base directory (see VENV)
#make pip-install install python packages in requirements.txt
#make git-config set local git configuration
export MAKEFILE_USAGE_COMMON
export MAKEFILE_USAGE
# doesn't seem to work
#.DEFAULT: build
# @echo running default
# $(MAKE) build
.PHONY: default
default:
@# putting this here instead of inline dep because otherwise check_makefile.sh will fail the target as build target doesn't exist in this Makefile.in
@$(MAKE) build
.PHONY: main
main:
@$(MAKE) build test
.PHONY: help
help:
@# this doesn't work because the macro insertion of a multiline literal breaks the line-based make format so we have to export to an env var instead of using natively
@# even the unescaped macro literal in a commented breaks make
@echo "$$MAKEFILE_USAGE_COMMON $$MAKEFILE_USAGE" | less -RFXig
.PHONY: usage
usage: help
@:
# catchall - any unrecognized target will print usage
%::
@{ \
echo Unrecognized option $@; \
echo; \
$(MAKE) usage; \
} | less -RFXig
.PHONY: quick
quick:
QUICK=1 $(MAKE) build
.PHONY: submodules
submodules:
git submodule update --init --recursive
.PHONY: git-clean
git-clean:
@git clean -n -d
@printf "\n\n%s" "If you're happy with this list, run:"
@printf "\n\n%s\n\n" "git clean -f -d"
.PHONY: gitignore
gitignore:
$(BASH_TOOLS)/update_gitignore.io.sh
.PHONY: btest
btest: bash-test
5 years ago
@:
.PHONY: bash-test
bash-test:
$(BASH_TOOLS)/check_all.sh
.PHONY: push
push:
git push
.PHONY: system-packages
system-packages: submodules
if [ -x /sbin/apk ]; then $(MAKE) apk-packages; fi
if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages; fi
if [ -x /usr/bin/yum ]; then $(MAKE) yum-packages; fi
if [ -x /usr/local/bin/brew -a `uname` = Darwin ]; then $(MAKE) homebrew-packages; fi
.PHONY: system-packages-perl
system-packages-perl: system-packages
if [ -x /sbin/apk ]; then $(MAKE) apk-packages-perl; fi
if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages-perl; fi
if [ -x /usr/bin/yum ]; then $(MAKE) yum-packages-perl; fi
.PHONY: system-packages-python
system-packages-python: system-packages
if [ -x /sbin/apk ]; then $(MAKE) apk-packages-python; fi
if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages-python; fi
if [ -x /usr/bin/yum ]; then $(MAKE) yum-packages-python; fi
.PHONY: apk-packages
apk-packages:
# not portable in Alpine sh
#for x in apk-packages{,-perl,-python}{,-dev}.txt; do \
for x in apk-packages.txt apk-packages-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/apk_install_packages.sh "setup/$$x"; \
fi; \
done
#for x in apk-packages-{optional,cpan,pip}.txt; do \
for x in apk-packages-optional.txt; do \
if [ -f "setup/$$x" ]; then \
NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/apk_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: apk-packages-perl
apk-packages-perl:
for x in apk-packages-perl.txt apk-packages-perl-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/apk_install_packages.sh "setup/$$x"; \
fi; \
done
#for x in apk-packages-{optional,cpan,pip}.txt; do \
for x in apk-packages-cpan.txt; do \
if [ -f "setup/$$x" ] && [ -z "$(PERLBREW_PERL)" ]; then \
NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/apk_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: apk-packages-python
apk-packages-python:
for x in apk-packages-python.txt apk-packages-python-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/apk_install_packages.sh "setup/$$x"; \
fi; \
done
for x in apk-packages-pip.txt; do \
if [ -f "setup/$$x" ] && [ -z "$(PYTHON_VIRTUALENV)" ]; then \
NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/apk_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: apt-packages
apt-packages:
#for x in deb-packages{,-perl,-python}{,-dev}.txt; do \
for x in deb-packages.txt deb-packages-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/apt_install_packages.sh "setup/$$x"; \
fi; \
done
#for x in deb-packages-{optional,cpan,pip}.txt; do \
for x in deb-packages-optional.txt; do \
if [ -f "setup/$$x" ]; then \
NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/apt_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: apt-packages-perl
apt-packages-perl:
for x in deb-packages-perl.txt deb-packages-perl-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/apt_install_packages.sh "setup/$$x"; \
fi; \
done
for x in deb-packages-cpan.txt; do \
if [ -f "setup/$$x" ] && \
[ -z "$(PERLBREW_PERL)" ] && \
[ -z "$(GOOGLE_CLOUD_SHELL)" ]; then \
NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/apt_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: apt-packages-python
apt-packages-python:
for x in deb-packages-python.txt deb-packages-python-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/apt_install_packages.sh "setup/$$x"; \
fi; \
done
for x in deb-packages-pip.txt; do \
if [ -f "setup/$$x" ] && \
[ -z "$(PYTHON_VIRTUALENV)" ] && \
[ -z "$(GOOGLE_CLOUD_SHELL)" ]; then \
NO_FAIL=1 NO_UPDATE=1 $(BASH_TOOLS)/apt_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: yum-packages
yum-packages:
$(BASH_TOOLS)/setup/install_epel_repo.sh
# installing packages individually to catch package install failure, otherwise yum succeeds even if it misses a package
for x in rpm-packages.txt rpm-packages-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/yum_install_packages.sh "setup/$$x"; \
fi; \
done
for x in rpm-packages-optional.txt; do \
if [ -f "setup/$$x" ]; then \
NO_FAIL=1 $(BASH_TOOLS)/yum_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: yum-packages-perl
yum-packages-perl:
# installing packages individually to catch package install failure, otherwise yum succeeds even if it misses a package
for x in rpm-packages-perl.txt rpm-packages-perl-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/yum_install_packages.sh "setup/$$x"; \
fi; \
done
for x in rpm-packages-cpan.txt; do \
if [ -f "setup/$$x" ] && [ -z "$(PERLBREW_PERL)" ]; then \
NO_FAIL=1 $(BASH_TOOLS)/yum_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: yum-packages-python
yum-packages-python:
# installing packages individually to catch package install failure, otherwise yum succeeds even if it misses a package
for x in rpm-packages-python.txt rpm-packages-python-dev.txt; do \
if [ -f "setup/$$x" ]; then \
$(BASH_TOOLS)/yum_install_packages.sh "setup/$$x"; \
fi; \
done
for x in rpm-packages-pip.txt; do \
if [ -f "setup/$$x" ] && [ -z "$(PYTHON_VIRTUALENV)" ]; then \
NO_FAIL=1 $(BASH_TOOLS)/yum_install_packages.sh "setup/$$x"; \
fi; \
done
.PHONY: homebrew-packages
homebrew-packages:
# Fails if any of the packages are already installed, ignore and continue - if it's a problem the latest build steps will fail with missing headers
if test -f setup/brew-packages.txt; then \
NO_FAIL=1 $(BASH_TOOLS)/brew_install_packages.sh setup/brew-packages.txt; \
fi
@# fix for OpenSSL 1.0 -> 1.1 library linkage breaking python -c 'import hashlib', which break pips, eg:
@# https://stackoverflow.com/questions/20399331/error-importing-hashlib-with-python-2-7-but-not-with-2-6
$(BASH_TOOLS)/setup/brew_fix_openssl_dependencies.sh
.PHONY: system-packages-remove
system-packages-remove:
if [ -x /sbin/apk ]; then $(MAKE) apk-packages-remove; fi
if [ -x /usr/bin/apt-get ]; then $(MAKE) apt-packages-remove; fi
if [ -x /usr/bin/yum ]; then $(MAKE) yum-packages-remove; fi
.PHONY: apk-packages-remove
apk-packages-remove:
for x in lib pylib lib-java; do test -f "$$x" && pushd "$$X" && $(MAKE) apk-packages-remove; popd; done; :
for x in apk-packages-{,perl-,python-}dev.txt; do \
if [ -f "setup/$$x" ]; then \
NO_FAIL=1 $(BASH_TOOLS)/apk_remove_packages.sh "setup/$xx"; \
fi; \
done
$(SUDO) rm -fr /var/cache/apk/*
.PHONY: apt-packages-remove
apt-packages-remove:
for x in lib pylib lib-java; do test -f "$$x" && pushd "$$X" && $(MAKE) apt-packages-remove; popd; done; :
for x in deb-packages-{,perl-,python-}dev.txt; do \
if [ -f "setup/$$x" ]; then \
NO_FAIL=1 $(BASH_TOOLS)/apt_remove_packages.sh "setup/$xx"; \
fi; \
done
.PHONY: yum-packages-remove
yum-packages-remove:
for x in lib pylib lib-java; do test -f "$$x" && pushd "$$X" && $(MAKE) yum-packages-remove; popd; done; :
for x in rpm-packages-{,perl-,python-}dev.txt; do \
if [ -f "setup/$$x" ]; then \
NO_FAIL=1 $(BASH_TOOLS)/yum_remove_packages.sh "setup/$xx"; \
fi; \
done
.PHONY: cpan
cpan:
find . -name 'cpan-requirements*.txt' | xargs $(BASH_TOOLS)/perl_cpanm_install_if_absent.sh
.PHONY: pip
pip:
find . -name 'requirements.txt' | xargs $(BASH_TOOLS)/python_pip_install_if_absent.sh
.PHONY: fatpacks
fatpacks:
$(BASH_TOOLS)/perl_generate_fatpacks.sh *.pl
@echo
if [ -d lib/resources ]; then \
cp -av lib/resources fatpacks/; \
fi
@if $(MAKE) -n fatpacks-local >/dev/null 2>&1; then \
echo; \
echo "fatpacks-local target detected, running:"; \
$(MAKE) fatpacks-local; \
fi
@echo
tar czvf fatpacks.tar.gz "$(FATPACKS_DIR)"
@echo
@echo "Generated fatpacks.tar.gz containing $(FATPACKS_DIR)/ directory of perl scripts with all dependencies bundled"
.PHONY: fatpack
fatpack: fatpacks
@:
.PHONY: python-compile
python-compile:
$(BASH_TOOLS)/python_compile.sh
.PHONY: pycompile
pycompile: python-compile
@:
# =======================
# Nice tricks for pure Python projects
# - borrowed from https://gist.github.com/bsmith89/c6811893c1cbd2a72cc1d144a197bef2#file-makefile
#VENV = .venv
#export VIRTUAL_ENV := $(abspath ${VENV})
# putting the venv/bin at the start of the path means that the venv python will be called
# and the venv libraries used automatically, so no need to 'source .venv/bin/activate' first
# although it misses the hash flush 'hash -r' that the venv activate script does
#export PATH := ${VIRTUAL_ENV}/bin:${PATH}
#${VENV}:
# python3 -m venv "$@"
#pip-install: requirements.txt | ${VENV}
# pip install --upgrade -r requirements.txt
# =======================
.PHONY: sonar
sonar:
sonar-scanner
.PHONY: update
update: update2
@# putting this here instead of inline dep because otherwise check_makefile.sh will fail the target as build target doesn't exist in this Makefile.in
@$(MAKE) build
.PHONY: update2
update2: update-no-recompile
5 years ago
@:
.PHONY: update-no-recompile
update-no-recompile:
git pull
$(MAKE) submodules
.PHONY: update-submodules
update-submodules:
git submodule update --init --remote
.PHONY: updatem
updatem: update-submodules
5 years ago
@:
.PHONY: docker-run
docker-run:
docker run -ti --rm ${DOCKER_IMAGE} ${ARGS}
.PHONY: run
run: docker-run
5 years ago
@:
.PHONY: docker-mount
docker-mount:
# --privileged=true is needed to be able to:
# mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk
docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE} bash -c "cd /code; exec bash"
.PHONY: docker-mount-alpine
docker-mount-alpine:
# --privileged=true is needed to be able to:
# mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk
docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE}:alpine bash -c "cd /code; exec bash"
.PHONY: docker-mount-debian
docker-mount-debian:
# --privileged=true is needed to be able to:
# mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk
docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE}:debian bash -c "cd /code; exec bash"
.PHONY: docker-mount-centos
docker-mount-centos:
# --privileged=true is needed to be able to:
# mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk
docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE}:centos bash -c "cd /code; exec bash"
.PHONY: docker-mount-ubuntu
docker-mount-ubuntu:
# --privileged=true is needed to be able to:
# mount -t tmpfs -o size=1m tmpfs /mnt/ramdisk
docker run -ti --rm --privileged=true -v $$PWD:/code ${DOCKER_IMAGE}:ubuntu bash -c "cd /code; exec bash"
.PHONY: mount
mount: docker-mount
5 years ago
@:
.PHONY: mount-alpine
mount-alpine: docker-mount-alpine
5 years ago
@:
.PHONY: mount-debian
mount-debian: docker-mount-debian
5 years ago
@:
.PHONY: mount-centos
mount-centos: docker-mount-centos
5 years ago
@:
.PHONY: mount-ubuntu
mount-ubuntu: docker-mount-ubuntu
5 years ago
@:
# checks dockerhub build status for this repo - needs check_dockerhub_repo_build_status.py from Advanced Nagios Plugins Collection to be in $PATH
.PHONY: dockerhub-status
dockerhub-status:
check_dockerhub_repo_build_status.py -r "$(DOCKER_IMAGE)"
# For quick testing only - for actual Dockerfile builds see https://hub.docker.com/u/harisekhon and Dockerfiles source repo https://github.com/harisekhon/Dockerfiles
.PHONY: docker-alpine
docker-alpine:
$(BASH_TOOLS)/docker_mount_build_exec.sh alpine
.PHONY: docker-debian
docker-debian:
$(BASH_TOOLS)/docker_mount_build_exec.sh debian
.PHONY: docker-centos
docker-centos:
$(BASH_TOOLS)/docker_mount_build_exec.sh centos
.PHONY: docker-ubuntu
docker-ubuntu:
$(BASH_TOOLS)/docker_mount_build_exec.sh ubuntu
.PHONY: travis
travis:
@source $(BASH_TOOLS)/.bash.d/network.sh; browser "https://travis-ci.org/$(REPO)"
.PHONY: travis-log
travis-log:
travis_last_log.py --failed $(REPO)
.PHONY: travis-debug
travis-debug:
travis_debug_session.py $(REPO)
.PHONY: browse
browse:
@source $(BASH_TOOLS)/.bash.d/network.sh; browser "https://github.com/$(REPO)"
.PHONY: commitcount
commitcount:
@# interestingly, even on 10,000 commit repos, there are no duplicate short hashes shown from:
@# git log --all --pretty=format:"%h" | sort | uniq -d
@git log --all --pretty=format:"%h" | wc -l
.PHONY: github
github: browse
@:
.PHONY: github-url
github-url:
@source $(BASH_TOOLS)/.bash.d/functions.sh; echo "https://github.com/$(REPO)" | tee /dev/stderr | tr -d '\n' | paste_clipboard
.PHONY: readme
readme:
@source $(BASH_TOOLS)/.bash.d/network.sh; browser "https://github.com/$(REPO)/blob/master/README.md"
.PHONY: issues
issues:
@source $(BASH_TOOLS)/.bash.d/network.sh; browser "https://github.com/$(REPO)/issues"
.PHONY: github
dockerhub:
@source $(BASH_TOOLS)/.bash.d/network.sh; browser "https://hub.docker.com/u/harisekhon"
.PHONY: dockerhub-url
dockerhub-url:
@source $(BASH_TOOLS)/.bash.d/functions.sh; echo "https://hub.docker.com/u/harisekhon" | tee /dev/stderr | tr -d '\n' | paste_clipboard
.PHONY: startrack
startrack:
@echo "Don't run this too much, you will hit an API limit against your IP"
@source $(BASH_TOOLS)/.bash.d/network.sh; \
browser "https://seladb.github.io/StarTrack-js/?\
u=$$(sed 's/\/.*//' <<< "$(REPO)")\
&r=$$(sed 's/.*\///' <<< "$(REPO)")"
.PHONY: star
star: startrack
@:
.PHONY: allstars
allstars:
@echo "Takes a while, don't run this all the time or you will an API limit against your IP"
@REPOS="Nagios-Plugins Dockerfiles DevOps-Python-tools DevOps-Perl-tools DevOps-Bash-Tools Nagios-Plugin-Kafka HAProxy-configs"; \
source $(BASH_TOOLS)/.bash.d/network.sh; \
browser "https://seladb.github.io/StarTrack-js/?\
$$(\
for repo in $$REPOS; do \
printf "%s" "&u=HariSekhon&r=$$repo"; \
done | \
sed 's/\&//'\
)"
.PHONY: ls-files
ls-files:
@echo $(CODE_FILES) | tr ' ' '\n' | sort
.PHONY: files
files: ls-files
@:
.PHONY: ls-code
ls-code:
@# TODO: port out my code and non-code lists and make this more generic
@$(MAKE) ls-files | grep -v -e '^\./\.' -e LICENSE -e '.*\.md' -e '.*\.txt' `for x in $(CONF_FILES) $(shell git submodule | awk '{print $$2}'); do echo "-e $$x"; done`
@$(MAKE) ls-files | grep '\.bash' || :
.PHONY: lscode
lscode: ls-code
@:
.PHONY: wc
wc:
@# CODE_FILES := definitions in Makefiles must not be quoted or will get wc error 'open: File name too long'
@wc -l $(CODE_FILES)
@printf "Total Files: "
@tr ' ' '\n' <<< "$(CODE_FILES)" | wc -l
.PHONY: wc2
wc2:
@printf "Total Files: "
@tr ' ' '\n' <<< "$(CODE_FILES)" | wc -l
@printf "Total line count without # comments:"
@sed 's/#.*//;/^[[:space:]]*$$/d' $(CODE_FILES) | wc -l
.PHONY: wc-code
wc-code:
@$(MAKE) ls-code | xargs wc -l
@printf "Total code files: "
@$(MAKE) ls-code | wc -l
.PHONY: wccode
wccode: wc-code
@:
.PHONY: wc-code2
wc-code2:
@printf "Total code files: "
@$(MAKE) ls-code | wc -l
@printf "Total line count without # comments: "
@$(MAKE) ls-code | xargs sed 's/#.*//;/^[[:space:]]*$$/d' | wc -l
.PHONY: wccode
wccode2: wc-code2
@:
# finds .swp, would need to port out code lists
#.PHONY: wcall
#wcall:
# find . -type f --not -path '*.git*' -exec cat {} \; | wc -l
#
#.PHONY: wcall
#wcall:
# find . -type f -not -path '*.git*' -exec sed 's/#.*//;/^[[:space:]]*$$/d' {} \; | wc -l