Skip to content

Commit ad16542

Browse files
amenocalamenocal
committed
add JSON output option for repo statistics script
1 parent 62a2f2a commit ad16542

1 file changed

Lines changed: 187 additions & 45 deletions

File tree

‎gh-repo-stats‎

Lines changed: 187 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ SLEEP_RETRY_COUNT='15' # Number of times to try to sleep
6060
SLEEP_COUNTER='0' # Counter of how many times we have gone to sleep
6161
EXISTING_FILE='0' # Check if a file already exists
6262
OUTPUT="${OUTPUT_PARAM:-CSV}" # Output type CSV or Table
63+
JSON_OUTPUT=0 # Enable JSON output (0=disabled, 1=enabled)
6364
REPO_LIST_ARRAY=() # Array of repo names to be analyzed
6465
GITHUB_TOKEN_TYPE=${GITHUB_TOKEN_TYPE:-user} # Whether 'user' or 'app' PAT
6566
VERSION="cloud"
@@ -83,6 +84,8 @@ Options:
8384
-H, --hostname : The GitHub hostname for the request
8485
Default: github.com
8586
-i, --input : Set path to a file with a list of organizations to scan, one per line, newline delimited
87+
-J, --json : Output in JSON format instead of CSV
88+
Produces a .json file conforming to json-schema.md
8689
-o, --org : Name of the GitHub Organization to be analyzed
8790
-O, --output : Format of output, can either be "CSV" or "Table"
8891
Default: $OUTPUT
@@ -163,6 +166,10 @@ while (( "$#" )); do
163166
ORG_NAME=$2
164167
shift 2
165168
;;
169+
-J|--json)
170+
JSON_OUTPUT=1
171+
shift
172+
;;
166173
-rl|--repo-list)
167174
REPO_LIST_FILE=$2
168175
shift 2
@@ -321,7 +328,31 @@ Footer() {
321328
echo "######################################################"
322329
echo "The script has completed"
323330
echo ""
324-
if [[ ${OUTPUT} != "CSV" ]]; then
331+
if [[ ${JSON_OUTPUT} -eq 1 ]]; then
332+
################################
333+
# Assemble final JSON output #
334+
################################
335+
REPO_FILE_COUNT=$(find "${JSON_TEMP_DIR}" -name 'repo-*.json' -type f | wc -l | tr -d ' ')
336+
if [[ "${REPO_FILE_COUNT}" -gt 0 ]]; then
337+
# Use find+sort+xargs to avoid ARG_MAX limits for large orgs
338+
# Write to temp file first for atomic output (don't truncate existing file on failure)
339+
JSON_TEMP_OUTPUT="${JSON_TEMP_DIR}/final-output.json"
340+
find "${JSON_TEMP_DIR}" -name 'repo-*.json' -type f | sort | xargs cat | jq -s '{ repos: . }' > "${JSON_TEMP_OUTPUT}"
341+
ERROR_CODE=$?
342+
if [ "${ERROR_CODE}" -ne 0 ]; then
343+
echo "ERROR! Failed to assemble final JSON file: ${JSON_FILE_NAME}"
344+
else
345+
mv "${JSON_TEMP_OUTPUT}" "${JSON_FILE_NAME}"
346+
echo "Results file:[${JSON_FILE_NAME}]"
347+
fi
348+
else
349+
# No repos found, write empty structure
350+
echo '{ "repos": [] }' > "${JSON_FILE_NAME}"
351+
echo "Results file:[${JSON_FILE_NAME}] (no repositories found)"
352+
fi
353+
# Clean up temp directory (also handled by trap, but be explicit)
354+
rm -rf "${JSON_TEMP_DIR}"
355+
elif [[ ${OUTPUT} != "CSV" ]]; then
325356
column -t -s',' "${OUTPUT_FILE_NAME}"
326357
rm -f "${OUTPUT_FILE_NAME}"
327358
else
@@ -339,30 +370,54 @@ GenerateFiles() {
339370
# Get datestring YYYYMMDDHHMM
340371
DATE=$(date +%Y%m%d%H%M)
341372

373+
###############################
374+
# Handle JSON output mode #
375+
###############################
376+
if [[ ${JSON_OUTPUT} -eq 1 ]]; then
377+
JSON_FILE_NAME="$ORG_NAME-all_repos-$DATE.json"
378+
JSON_TEMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/gh-repo-stats-json.XXXXXX")
379+
JSON_REPO_COUNTER=0
380+
381+
# Ensure temp directory is cleaned up on exit, interrupt, or termination
382+
# shellcheck disable=SC2064
383+
trap "rm -rf '${JSON_TEMP_DIR}'" EXIT INT TERM
384+
385+
######################################################
386+
# Need to see if there is a file that already exists #
387+
######################################################
388+
EXISTING_JSON_CMD=$(find . -name "$ORG_NAME-all_repos-*.json" |grep . 2>&1)
389+
ERROR_CODE=$?
390+
if [ "${ERROR_CODE}" -eq 0 ]; then
391+
JSON_FILE_NAME="${EXISTING_JSON_CMD:2}"
392+
fi
393+
fi
394+
342395
####################
343396
# Create File Name #
344397
####################
345398
# Example: MyOrg-all_repos-201901041059.csv
346-
OUTPUT_FILE_NAME="$ORG_NAME-all_repos-$DATE.csv"
399+
if [[ ${JSON_OUTPUT} -ne 1 ]]; then
400+
OUTPUT_FILE_NAME="$ORG_NAME-all_repos-$DATE.csv"
347401

348-
######################################################
349-
# Need to see if there is a file that already exists #
350-
######################################################
351-
EXISTING_FILE_CMD=$(find . -name "$ORG_NAME-all_repos-*" |grep . 2>&1)
402+
######################################################
403+
# Need to see if there is a file that already exists #
404+
######################################################
405+
EXISTING_FILE_CMD=$(find . -name "$ORG_NAME-all_repos-*" |grep . 2>&1)
352406

353-
#######################
354-
# Load the error code #
355-
#######################
356-
ERROR_CODE=$?
407+
#######################
408+
# Load the error code #
409+
#######################
410+
ERROR_CODE=$?
357411

358-
##############################
359-
# Check the shell for errors #
360-
##############################
361-
if [ "${ERROR_CODE}" -eq 0 ]; then
362-
# There is already file
363-
# Going to use and append
364-
OUTPUT_FILE_NAME="${EXISTING_FILE_CMD:2}"
365-
EXISTING_FILE=1
412+
##############################
413+
# Check the shell for errors #
414+
##############################
415+
if [ "${ERROR_CODE}" -eq 0 ]; then
416+
# There is already file
417+
# Going to use and append
418+
OUTPUT_FILE_NAME="${EXISTING_FILE_CMD:2}"
419+
EXISTING_FILE=1
420+
fi
366421
fi
367422

368423
if [[ ${ANALYZE_CONFLICTS} -eq 1 ]]; then
@@ -426,7 +481,7 @@ GenerateFiles() {
426481
#########################################
427482
# Only add header if were not appending #
428483
#########################################
429-
if [ "${EXISTING_FILE}" -ne 1 ]; then
484+
if [[ ${JSON_OUTPUT} -ne 1 ]] && [ "${EXISTING_FILE}" -ne 1 ]; then
430485
#############################
431486
# Create Header in the file #
432487
#############################
@@ -828,27 +883,110 @@ ParseRepos() {
828883
#################################################################
829884
# Need to check if this repo has already been parsed in the doc #
830885
#################################################################
831-
grep "${OWNER},${REPO_NAME}," "${OUTPUT_FILE_NAME}" >/dev/null 2>&1
832-
833-
#######################
834-
# Load the error code #
835-
#######################
836-
ERROR_CODE=$?
837-
838-
##############################
839-
# Check the shell for errors #
840-
##############################
841-
if [ ${ERROR_CODE} -eq 0 ]; then
842-
# Found this in the csv already
843-
echo "Repo:[${OWNER}/${REPO_NAME}] has previously been analyzed, moving on..."
844-
elif [ ${ERROR_CODE} -ne 0 ] && [[ -z "${REPO_LIST_FILE}" ]]; then
886+
if [[ ${JSON_OUTPUT} -eq 1 ]]; then
887+
# In JSON mode, skip CSV de-dupe check and always parse
845888
echo "Analyzing Repo: ${REPO_NAME}"
846889
ParseRepoData "${REPO_DATA}"
890+
else
891+
grep "${OWNER},${REPO_NAME}," "${OUTPUT_FILE_NAME}" >/dev/null 2>&1
892+
893+
#######################
894+
# Load the error code #
895+
#######################
896+
ERROR_CODE=$?
897+
898+
##############################
899+
# Check the shell for errors #
900+
##############################
901+
if [ ${ERROR_CODE} -eq 0 ]; then
902+
# Found this in the csv already
903+
echo "Repo:[${OWNER}/${REPO_NAME}] has previously been analyzed, moving on..."
904+
elif [ ${ERROR_CODE} -ne 0 ] && [[ -z "${REPO_LIST_FILE}" ]]; then
905+
echo "Analyzing Repo: ${REPO_NAME}"
906+
ParseRepoData "${REPO_DATA}"
907+
fi
847908
fi
848909
fi
849910
done
850911
}
851912
################################################################################
913+
#### Function GenerateJSONObject ###############################################
914+
GenerateJSONObject() {
915+
########################################################
916+
# Build a JSON object for this repo from existing data #
917+
########################################################
918+
((JSON_REPO_COUNTER++))
919+
920+
REPO_JSON=$(jq -n \
921+
--arg org "$ORG_NAME" \
922+
--arg name "$REPO_NAME" \
923+
--arg url "$URL" \
924+
--argjson isFork "$IS_FORK" \
925+
--argjson isArchived "$IS_ARCHIVED" \
926+
--argjson diskUsage "${REPO_SIZE_KB:-0}" \
927+
--argjson sizeMB "${REPO_SIZE:-0}" \
928+
--argjson hasWikiEnabled "$HAS_WIKI" \
929+
--arg createdAt "$CREATED_AT" \
930+
--arg updatedAt "$UPDATED_AT" \
931+
--arg pushedAt "$PUSHED_AT" \
932+
--argjson collaborators "${COLLABORATOR_CT:-0}" \
933+
--argjson branches "${BRANCH_CT:-0}" \
934+
--argjson tags "${TAG_CT:-0}" \
935+
--argjson branchProtections "${PROTECTED_BRANCH_CT:-0}" \
936+
--argjson issues "${ISSUE_CT:-0}" \
937+
--argjson pullRequests "${PR_CT:-0}" \
938+
--argjson milestones "${MILESTONE_CT:-0}" \
939+
--argjson releases "${RELEASE_CT:-0}" \
940+
--argjson projects "${PROJECT_CT:-0}" \
941+
--argjson discussions "${DISCUSSION_CT:-0}" \
942+
--argjson commitComments "${COMMIT_COMMENT_CT:-0}" \
943+
--argjson issueEvents "${ISSUE_EVENT_CT:-0}" \
944+
'{
945+
org: $org,
946+
name: $name,
947+
url: $url,
948+
isFork: $isFork,
949+
isArchived: $isArchived,
950+
diskUsage: $diskUsage,
951+
sizeMB: $sizeMB,
952+
hasWikiEnabled: $hasWikiEnabled,
953+
createdAt: $createdAt,
954+
updatedAt: $updatedAt,
955+
pushedAt: $pushedAt,
956+
collaborators: $collaborators,
957+
branches: $branches,
958+
tags: $tags,
959+
branchProtections: $branchProtections,
960+
issues: $issues,
961+
pullRequests: $pullRequests,
962+
milestones: $milestones,
963+
releases: $releases,
964+
projects: $projects,
965+
discussions: $discussions,
966+
commitComments: $commitComments,
967+
issueEvents: $issueEvents
968+
}')
969+
970+
ERROR_CODE=$?
971+
if [ "${ERROR_CODE}" -ne 0 ]; then
972+
echo "ERROR! Failed to generate JSON for repo: ${ORG_NAME}/${REPO_NAME}"
973+
return 1
974+
fi
975+
976+
################################
977+
# Write JSON to temp directory #
978+
################################
979+
echo "${REPO_JSON}" > "${JSON_TEMP_DIR}/repo-$(printf '%06d' ${JSON_REPO_COUNTER}).json"
980+
981+
ERROR_CODE=$?
982+
if [ "${ERROR_CODE}" -ne 0 ]; then
983+
echo "ERROR! Failed to write JSON file for repo: ${ORG_NAME}/${REPO_NAME}"
984+
return 1
985+
fi
986+
987+
Debug "Generated JSON for repo: ${ORG_NAME}/${REPO_NAME}"
988+
}
989+
################################################################################
852990
#### Function ParseRepoData ####################################################
853991
ParseRepoData() {
854992
# Pull in the repos data block
@@ -930,20 +1068,24 @@ ParseRepoData() {
9301068
########################
9311069
# Write it to the file #
9321070
########################
933-
echo "${ORG_NAME},${REPO_NAME},${IS_EMPTY}${PUSHED_AT},${UPDATED_AT},${IS_FORK},${IS_ARCHIVED},${REPO_SIZE},${RECORD_CT},${COLLABORATOR_CT},${PROTECTED_BRANCH_CT},${PR_REVIEW_CT},${MILESTONE_CT},${ISSUE_CT},${PR_CT},${PR_REVIEW_COMMENT_CT},${COMMIT_COMMENT_CT},${ISSUE_COMMENT_CT},${ISSUE_EVENT_CT},${RELEASE_CT},${PROJECT_CT},${BRANCH_CT},${TAG_CT},${DISCUSSION_CT},${HAS_WIKI},${URL},${MIGRATION_ISSUE},${CREATED_AT}" >>"${OUTPUT_FILE_NAME}"
1071+
if [[ ${JSON_OUTPUT} -eq 1 ]]; then
1072+
GenerateJSONObject
1073+
else
1074+
echo "${ORG_NAME},${REPO_NAME},${IS_EMPTY}${PUSHED_AT},${UPDATED_AT},${IS_FORK},${IS_ARCHIVED},${REPO_SIZE},${RECORD_CT},${COLLABORATOR_CT},${PROTECTED_BRANCH_CT},${PR_REVIEW_CT},${MILESTONE_CT},${ISSUE_CT},${PR_CT},${PR_REVIEW_COMMENT_CT},${COMMIT_COMMENT_CT},${ISSUE_COMMENT_CT},${ISSUE_EVENT_CT},${RELEASE_CT},${PROJECT_CT},${BRANCH_CT},${TAG_CT},${DISCUSSION_CT},${HAS_WIKI},${URL},${MIGRATION_ISSUE},${CREATED_AT}" >>"${OUTPUT_FILE_NAME}"
9341075

935-
#######################
936-
# Load the error code #
937-
#######################
938-
# shellcheck disable=SC2320
939-
ERROR_CODE=$?
1076+
#######################
1077+
# Load the error code #
1078+
#######################
1079+
# shellcheck disable=SC2320
1080+
ERROR_CODE=$?
9401081

941-
##############################
942-
# Check the shell for errors #
943-
##############################
944-
if [ "${ERROR_CODE}" -ne 0 ]; then
945-
echo "ERROR! Failed to write output to file:[${OUTPUT_FILE_NAME}]"
946-
exit 1
1082+
##############################
1083+
# Check the shell for errors #
1084+
##############################
1085+
if [ "${ERROR_CODE}" -ne 0 ]; then
1086+
echo "ERROR! Failed to write output to file:[${OUTPUT_FILE_NAME}]"
1087+
exit 1
1088+
fi
9471089
fi
9481090

9491091
##############################

0 commit comments

Comments
 (0)