You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
142 lines
4.6 KiB
Bash
142 lines
4.6 KiB
Bash
#!/usr/bin/env bash
|
|
# vim:ts=4:sts=4:sw=4:et
|
|
#
|
|
# Author: Hari Sekhon
|
|
# Date: 2020-01-23 15:08:10 +0000 (Thu, 23 Jan 2020)
|
|
#
|
|
# https://github.com/harisekhon/bash-tools
|
|
#
|
|
# License: see accompanying Hari Sekhon LICENSE file
|
|
#
|
|
# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish
|
|
#
|
|
# https://www.linkedin.com/in/harisekhon
|
|
#
|
|
|
|
# Script to fetch Yarn apps from Cloudera Manager
|
|
#
|
|
# combine with jq commands to extract just the list of SQL queries from the rich json output
|
|
#
|
|
# ./cloudera_manager_yarn_apps.sh | jq -r '.applications[]'
|
|
|
|
# Tested on Cloudera Enterprise 5.10
|
|
|
|
# Raw JSON output example:
|
|
#
|
|
# {
|
|
# "applications" : [ {
|
|
# "applicationId" : "application_1571433934948_123456",
|
|
# "name" : "HdfsReplication",
|
|
# "startTime" : "2020-01-19T18:17:52.257Z",
|
|
# "user" : "hive",
|
|
# "pool" : "root.users.hive",
|
|
# "state" : "RUNNING",
|
|
# "progress" : 95.0,
|
|
# "attributes" : {
|
|
# "failed_map_attempts" : "0",
|
|
# "failed_reduce_attempts" : "0",
|
|
# "failed_tasks_attempts" : "0",
|
|
# "killed_map_attempts" : "0",
|
|
# "killed_reduce_attempts" : "0",
|
|
# "killed_tasks_attempts" : "0",
|
|
# "map_progress" : "100.0",
|
|
# "maps_completed" : "20",
|
|
# "maps_pending" : "0",
|
|
# "maps_running" : "0",
|
|
# "maps_total" : "20",
|
|
# "new_map_attempts" : "0",
|
|
# "new_reduce_attempts" : "0",
|
|
# "new_tasks_attempts" : "0",
|
|
# "reduce_progress" : "100.0",
|
|
# "reduces_completed" : "0",
|
|
# "reduces_pending" : "0",
|
|
# "reduces_running" : "1",
|
|
# "reduces_total" : "1",
|
|
# "running_application_info_retrieval_time" : "8.97",
|
|
# "running_map_attempts" : "0",
|
|
# "running_reduce_attempts" : "1",
|
|
# "running_tasks_attempts" : "1",
|
|
# "successful_map_attempts" : "20",
|
|
# "successful_reduce_attempts" : "0",
|
|
# "successful_tasks_attempts" : "20"
|
|
# "tasks_completed" : "20",
|
|
# "tasks_pending" : "0",
|
|
# "tasks_running" : "1",
|
|
# "total_task_num" : "21",
|
|
# "tracking_url" : "https://<fqdn>:8090/proxy/application_1571433934948_123456/",
|
|
# "uberized" : "false",
|
|
# },
|
|
# "mr2AppInformation" : { }
|
|
# }, {
|
|
# "applicationId" : "application_1571433934948_123456",
|
|
# "name" : "oozie:launcher:T=shell:W=blah/blah:A=myuser-node:ID=0009168-200118163518563-oozie-oozi-W",
|
|
# "startTime" : "2020-01-23T14:40:48.442Z",
|
|
# "user" : "myuser",
|
|
# "pool" : "root.stream",
|
|
# "state" : "RUNNING",
|
|
# "progress" : 95.0,
|
|
# "attributes" : {
|
|
# "failed_map_attempts" : "0",
|
|
# "failed_reduce_attempts" : "0",
|
|
# "failed_tasks_attempts" : "0",
|
|
# "killed_map_attempts" : "0",
|
|
# "killed_reduce_attempts" : "0",
|
|
# "killed_tasks_attempts" : "0",
|
|
# "map_progress" : "100.0",
|
|
# "maps_completed" : "0",
|
|
# "maps_pending" : "0",
|
|
# "maps_running" : "1",
|
|
# "maps_total" : "1",
|
|
# "new_map_attempts" : "0",
|
|
# "new_reduce_attempts" : "0",
|
|
# "new_tasks_attempts" : "0",
|
|
# "reduce_progress" : "0.0",
|
|
# "reduces_completed" : "0",
|
|
# "reduces_pending" : "0",
|
|
# "reduces_running" : "0",
|
|
# "reduces_total" : "0",
|
|
# "running_application_info_retrieval_time" : "0.008",
|
|
# "running_map_attempts" : "1",
|
|
# "running_reduce_attempts" : "0",
|
|
# "running_tasks_attempts" : "1",
|
|
# "successful_map_attempts" : "0",
|
|
# "successful_reduce_attempts" : "0",
|
|
# "successful_tasks_attempts" : "0"
|
|
# "tasks_completed" : "0",
|
|
# "tasks_pending" : "0",
|
|
# "tasks_running" : "1",
|
|
# "total_task_num" : "1",
|
|
# "tracking_url" : "https://<fqdn>:8090/proxy/application_1571433934948_123456/",
|
|
# "uberized" : "false",
|
|
# },
|
|
# "mr2AppInformation" : { }
|
|
# },
|
|
# ...
|
|
# {
|
|
# "applicationId" : "application_1571433934948_123456",
|
|
# "name" : "my_biz_process.py",
|
|
# "startTime" : "2020-01-23T09:26:56.878Z",
|
|
# "endTime" : "2020-01-23T09:29:50.705Z",
|
|
# "user" : "myETLuser",
|
|
# "pool" : "root.users.myETLuser",
|
|
# "state" : "FINISHED",
|
|
# "progress" : 100.0,
|
|
# "attributes" : { },
|
|
# "mr2AppInformation" : { }
|
|
# } ],
|
|
# }
|
|
|
|
set -euo pipefail
|
|
[ -n "${DEBUG:-}" ] && set -x
|
|
srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# shellcheck disable=SC1090
|
|
. "$srcdir/lib/cloudera_manager.sh"
|
|
|
|
# defined in lib
|
|
# shellcheck disable=SC2154
|
|
echo "fetching queries up to now: $now_timestamp" >&2
|
|
|
|
"$srcdir/curl_auth.sh" -sS "$CLOUDERA_MANAGER/api/v7/clusters/$CLOUDERA_CLUSTER/services/yarn/yarnApplications?from=1970-01-01T00%3A00%3A00.000Z&to=$now_timestamp&filter="
|
|
echo
|