Enhance README and script to support field configuration for output customization

sukumarp2022 · sukumarp2022 · commit 0a8de68a5feb · 2025-12-30T18:01:06.000+05:30
diff --git a/README.md b/README.md
@@ -20,6 +20,10 @@ $ gh repo-stats --help
 Usage: gh repo-stats [options]
 
 Options:
+     -b, --batch                   : Number of repositories to process in parallel
+                                     Default: 1 (sequential processing)
+     -c, --config                  : Path to field configuration CSV file (Field_Name,Should_Fetch format)
+                                     If not provided, all fields are fetched
      -d, --debug                   : Enable Debug logging
      -e, --extra-page-size         : Set the pagination size for subsequent, paginated GraphQL queries; reduce if timeout occurs
                                      Default: 50
@@ -49,6 +53,7 @@ Description:
 Example:
   gh repo-stats -o my-org-name
   gh repo-stats -o my-org-name -H github.example.com
+  gh repo-stats -o my-org-name -c fields-config.csv
 ```
 
 ## Permissions
@@ -72,7 +77,7 @@ The permissions needed by `gh repo-stats` depends based on `-y, --token-type`:
 `gh repo-stats` produces either a visual table or `*.csv` file containing detailed information about various records within repositories.
 
 ```csv
-Org_Name,Repo_Name,Is_Empty,Last_Push,Last_Update,Latest_Commit_SHA,Latest_Committed_At,Latest_Commit_Branch,Default_Branch,isFork,isArchive,Repo_Size(mb),Record_Count,Collaborator_Count,Protected_Branch_Count,PR_Review_Count,Milestone_Count,Issue_Count,PR_Count,PR_Review_Comment_Count,Commit_Comment_Count,Issue_Comment_Count,Issue_Event_Count,Release_Count,Project_Count,Branch_Count,Tag_Count,Discussion_Count,Has_Wiki,Full_URL,Migration_Issue,Created
+Org_Name,Repo_Name,Is_Empty,Last_Push,Last_Update,Latest_Commit_SHA,Latest_Committed_At,Latest_Commit_Branch,Default_Branch,isFork,isArchive,Repo_Size_mb,Record_Count,Collaborator_Count,Protected_Branch_Count,PR_Review_Count,Milestone_Count,Issue_Count,PR_Count,PR_Review_Comment_Count,Commit_Comment_Count,Issue_Comment_Count,Issue_Event_Count,Release_Count,Project_Count,Branch_Count,Tag_Count,Discussion_Count,Has_Wiki,Full_URL,Migration_Issue,Created
 tinyfists,actions-experiments,false,2023-03-10T16:15:27Z,2022-10-28T19:38:34Z,abc123def456,2023-03-10T16:15:27Z,feature-branch,main,false,false,0,19,18,0,0,0,0,1,0,0,0,0,0,0,2,0,0,true,https://github.com/tinyfists/actions-experiments,FALSE,2020-01-01T13:37:00Z
 ```
 
@@ -89,7 +94,7 @@ tinyfists,actions-experiments,false,2023-03-10T16:15:27Z,2022-10-28T19:38:34Z,ab
 - `Default_Branch`: Default branch of the repository (e.g., main, master), or "No default branch" if none exists
 - `isFork`: Whether the repository is a fork
 - `isArchive`: Whether the repository is archived
-- `Repo_Size(mb)`: Size of the repository in megabytes
+- `Repo_Size_mb`: Size of the repository in megabytes
 - `Record_Count`: Number of database records this repository represents
 - `Collaborator_Count`: Number of users who have contributed to this repository
 - `Protected_Branch_Count`: Number of branch protection rules on this repository
@@ -112,3 +117,71 @@ tinyfists,actions-experiments,false,2023-03-10T16:15:27Z,2022-10-28T19:38:34Z,ab
   - 60,000 or more number of objects being imported
   - 1.5 GB or larger size on disk
 - `Created`: Date/time when the repository was created
+
+## Field Configuration
+
+You can customize which fields are included in the output by using a field configuration CSV file with the `-c` or `--config` option. This is useful when you only need specific fields and want to reduce the output size.
+
+### Configuration File Format
+
+Create a CSV file with the following format:
+
+```csv
+Field_Name,Should_Fetch
+Org_Name,true
+Repo_Name,true
+Is_Empty,false
+Last_Push,true
+...
+```
+
+- `Field_Name`: The name of the field (must match exactly)
+- `Should_Fetch`: `true` to include the field, `false` to exclude it
+
+### Example Configuration File
+
+A sample configuration file `fields-config.csv` is provided in the repository with all fields enabled. You can copy and modify it to suit your needs:
+
+```shell
+# Use custom field configuration
+gh repo-stats -o my-org-name -c fields-config.csv
+```
+
+### Available Fields
+
+| Field Name | Description |
+|------------|-------------|
+| `Org_Name` | Organization login |
+| `Repo_Name` | Repository name |
+| `Is_Empty` | Whether the repository is empty |
+| `Last_Push` | Date/time when a push was last made |
+| `Last_Update` | Date/time when an update was last made |
+| `Latest_Commit_SHA` | SHA of the most recent commit |
+| `Latest_Committed_At` | Date/time of the most recent commit |
+| `Latest_Commit_Branch` | Branch of the most recent commit |
+| `Default_Branch` | Default branch of the repository |
+| `isFork` | Whether the repository is a fork |
+| `isArchived` | Whether the repository is archived |
+| `Repo_Size_mb` | Size of the repository in MB |
+| `Record_Count` | Number of database records |
+| `Collaborator_Count` | Number of collaborators |
+| `Protected_Branch_Count` | Number of branch protection rules |
+| `PR_Review_Count` | Number of pull request reviews |
+| `Milestone_Count` | Number of milestones |
+| `Issue_Count` | Number of issues |
+| `PR_Count` | Number of pull requests |
+| `PR_Review_Comment_Count` | Number of PR review comments |
+| `Commit_Comment_Count` | Number of commit comments |
+| `Issue_Comment_Count` | Number of issue comments |
+| `Issue_Event_Count` | Number of issue events |
+| `Release_Count` | Number of releases |
+| `Project_Count` | Number of projects |
+| `Branch_Count` | Number of branches |
+| `Tag_Count` | Number of tags |
+| `Discussion_Count` | Number of discussions |
+| `Has_Wiki` | Whether wiki is enabled |
+| `Full_URL` | Repository URL |
+| `Migration_Issue` | Migration issue indicator |
+| `Created` | Repository creation date |
+
+**Note**: If no configuration file is specified or the file doesn't exist, all fields will be fetched by default.
diff --git a/gh-repo-stats b/gh-repo-stats
@@ -22,7 +22,7 @@
 # Default Branch (default branch of the repo, or "No default branch")
 # isFork
 # isArchived
-# Size(mb)
+# Size_mb
 # Total record count
 # Collaborator count
 # Protected branches
@@ -73,6 +73,8 @@ REPO_PAGE_SIZE=10
 EXTRA_PAGE_SIZE=50
 BATCH_SIZE=1                                  # Number of repos to process in parallel (1 = sequential)
 REPOS_TO_PROCESS=()                           # Global array to accumulate repos for batch processing
+FIELD_CONFIG_FILE=""                          # Path to field configuration CSV file
+DISABLED_FIELDS=""                            # Comma-separated list of disabled fields
 
 ################################################################################
 #### Function PrintUsage #######################################################
@@ -84,6 +86,8 @@ Usage: gh repo-stats [options]
 Options:
      -b, --batch                   : Number of repositories to process in parallel
                                      Default: $BATCH_SIZE (sequential processing)
+     -c, --config                  : Path to field configuration CSV file (Field_Name,Should_Fetch format)
+                                     If not provided, all fields are fetched
      -d, --debug                   : Enable Debug logging
      -e, --extra-page-size         : Set the pagination size for subsequent, paginated GraphQL queries; reduce if timeout occurs
                                      Default: $EXTRA_PAGE_SIZE
@@ -135,6 +139,10 @@ while (( "$#" )); do
       BATCH_SIZE=$2
       shift 2
       ;;
+    -c|--config)
+      FIELD_CONFIG_FILE=$2
+      shift 2
+      ;;
     -d|--debug)
       DEBUG=true
       export GH_DEBUG=api
@@ -221,6 +229,120 @@ Debug() {
   fi
 }
 ################################################################################
+#### Function LoadFieldConfig ##################################################
+LoadFieldConfig() {
+  # Initialize - by default all fields are enabled (DISABLED_FIELDS is empty)
+  DISABLED_FIELDS=""
+
+  # If no config file specified, all fields remain enabled
+  if [[ -z "${FIELD_CONFIG_FILE}" ]]; then
+    Debug "No field config file specified, all fields enabled"
+    return 0
+  fi
+
+  # Check if config file exists
+  if [[ ! -f "${FIELD_CONFIG_FILE}" ]]; then
+    echo "WARN: Field config file not found: ${FIELD_CONFIG_FILE}"
+    echo "Proceeding with all fields enabled"
+    return 0
+  fi
+
+  echo "Loading field configuration from: ${FIELD_CONFIG_FILE}"
+
+  # Read the config file and build DISABLED_FIELDS list
+  local line_num=0
+  local enabled_count=0
+  local disabled_count=0
+  while IFS=',' read -r field_name should_fetch || [[ -n "$field_name" ]]; do
+    ((line_num++))
+    # Skip header line
+    if [[ ${line_num} -eq 1 ]]; then
+      continue
+    fi
+    
+    # Trim whitespace
+    field_name=$(echo "${field_name}" | tr -d '[:space:]')
+    should_fetch=$(echo "${should_fetch}" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
+    
+    # Skip empty lines
+    if [[ -z "${field_name}" ]]; then
+      continue
+    fi
+
+    # Track disabled fields
+    if [[ "${should_fetch}" == "true" ]]; then
+      ((enabled_count++))
+      Debug "Field enabled: ${field_name}"
+    elif [[ "${should_fetch}" == "false" ]]; then
+      DISABLED_FIELDS="${DISABLED_FIELDS},${field_name},"
+      ((disabled_count++))
+      Debug "Field disabled: ${field_name}"
+    else
+      echo "WARN: Invalid value for field ${field_name}: ${should_fetch} (expected true/false)"
+    fi
+  done < "${FIELD_CONFIG_FILE}"
+
+  echo "Field configuration loaded: ${enabled_count} fields enabled, ${disabled_count} fields disabled"
+}
+################################################################################
+#### Function IsFieldEnabled ###################################################
+IsFieldEnabled() {
+  local field_name="$1"
+  # Check if field is in the disabled list
+  if [[ "${DISABLED_FIELDS}" == *",${field_name},"* ]]; then
+    return 1  # false - field is disabled
+  else
+    return 0  # true - field is enabled
+  fi
+}
+################################################################################
+#### Function BuildOutputRow ###################################################
+BuildOutputRow() {
+  # Build output row based on enabled fields
+  # Uses global variables for field values
+  local OUTPUT_ROW=""
+  
+  IsFieldEnabled "Org_Name" && OUTPUT_ROW="${OUTPUT_ROW}${ORG_NAME},"
+  IsFieldEnabled "Repo_Name" && OUTPUT_ROW="${OUTPUT_ROW}${REPO_NAME},"
+  if [[ "$(isCompatibleGHESVersion)" -eq "1" ]]; then
+    IsFieldEnabled "Is_Empty" && OUTPUT_ROW="${OUTPUT_ROW}${IS_EMPTY},"
+  fi
+  IsFieldEnabled "Last_Push" && OUTPUT_ROW="${OUTPUT_ROW}${PUSHED_AT},"
+  IsFieldEnabled "Last_Update" && OUTPUT_ROW="${OUTPUT_ROW}${UPDATED_AT},"
+  IsFieldEnabled "Latest_Commit_SHA" && OUTPUT_ROW="${OUTPUT_ROW}${LATEST_COMMIT_SHA},"
+  IsFieldEnabled "Latest_Committed_At" && OUTPUT_ROW="${OUTPUT_ROW}${LATEST_COMMITTED_AT},"
+  IsFieldEnabled "Latest_Commit_Branch" && OUTPUT_ROW="${OUTPUT_ROW}${LATEST_COMMIT_BRANCH},"
+  IsFieldEnabled "Default_Branch" && OUTPUT_ROW="${OUTPUT_ROW}${DEFAULT_BRANCH},"
+  IsFieldEnabled "isFork" && OUTPUT_ROW="${OUTPUT_ROW}${IS_FORK},"
+  IsFieldEnabled "isArchived" && OUTPUT_ROW="${OUTPUT_ROW}${IS_ARCHIVED},"
+  IsFieldEnabled "Repo_Size_mb" && OUTPUT_ROW="${OUTPUT_ROW}${REPO_SIZE},"
+  IsFieldEnabled "Record_Count" && OUTPUT_ROW="${OUTPUT_ROW}${RECORD_CT},"
+  IsFieldEnabled "Collaborator_Count" && OUTPUT_ROW="${OUTPUT_ROW}${COLLABORATOR_CT},"
+  IsFieldEnabled "Protected_Branch_Count" && OUTPUT_ROW="${OUTPUT_ROW}${PROTECTED_BRANCH_CT},"
+  IsFieldEnabled "PR_Review_Count" && OUTPUT_ROW="${OUTPUT_ROW}${PR_REVIEW_CT},"
+  IsFieldEnabled "Milestone_Count" && OUTPUT_ROW="${OUTPUT_ROW}${MILESTONE_CT},"
+  IsFieldEnabled "Issue_Count" && OUTPUT_ROW="${OUTPUT_ROW}${ISSUE_CT},"
+  IsFieldEnabled "PR_Count" && OUTPUT_ROW="${OUTPUT_ROW}${PR_CT},"
+  IsFieldEnabled "PR_Review_Comment_Count" && OUTPUT_ROW="${OUTPUT_ROW}${PR_REVIEW_COMMENT_CT},"
+  IsFieldEnabled "Commit_Comment_Count" && OUTPUT_ROW="${OUTPUT_ROW}${COMMIT_COMMENT_CT},"
+  IsFieldEnabled "Issue_Comment_Count" && OUTPUT_ROW="${OUTPUT_ROW}${ISSUE_COMMENT_CT},"
+  IsFieldEnabled "Issue_Event_Count" && OUTPUT_ROW="${OUTPUT_ROW}${ISSUE_EVENT_CT},"
+  IsFieldEnabled "Release_Count" && OUTPUT_ROW="${OUTPUT_ROW}${RELEASE_CT},"
+  IsFieldEnabled "Project_Count" && OUTPUT_ROW="${OUTPUT_ROW}${PROJECT_CT},"
+  IsFieldEnabled "Branch_Count" && OUTPUT_ROW="${OUTPUT_ROW}${BRANCH_CT},"
+  IsFieldEnabled "Tag_Count" && OUTPUT_ROW="${OUTPUT_ROW}${TAG_CT},"
+  IsFieldEnabled "Discussion_Count" && OUTPUT_ROW="${OUTPUT_ROW}${DISCUSSION_CT},"
+  IsFieldEnabled "Has_Wiki" && OUTPUT_ROW="${OUTPUT_ROW}${HAS_WIKI},"
+  IsFieldEnabled "Full_URL" && OUTPUT_ROW="${OUTPUT_ROW}${URL},"
+  IsFieldEnabled "Migration_Issue" && OUTPUT_ROW="${OUTPUT_ROW}${MIGRATION_ISSUE},"
+  IsFieldEnabled "Created" && OUTPUT_ROW="${OUTPUT_ROW}${CREATED_AT},"
+
+  # Remove trailing comma
+  OUTPUT_ROW="${OUTPUT_ROW%,}"
+  
+  echo "${OUTPUT_ROW}"
+}
+################################################################################
 #### Function Header ###########################################################
 Header() {
   echo ""
@@ -450,7 +572,48 @@ GenerateFiles() {
       IS_EMPTY_HEADER="";
     fi
 
-    echo "Org_Name,Repo_Name,${IS_EMPTY_HEADER}Last_Push,Last_Update,Latest_Commit_SHA,Latest_Committed_At,Latest_Commit_Branch,Default_Branch,isFork,isArchived,Repo_Size(mb),Record_Count,Collaborator_Count,Protected_Branch_Count,PR_Review_Count,Milestone_Count,Issue_Count,PR_Count,PR_Review_Comment_Count,Commit_Comment_Count,Issue_Comment_Count,Issue_Event_Count,Release_Count,Project_Count,Branch_Count,Tag_Count,Discussion_Count,Has_Wiki,Full_URL,Migration_Issue,Created" >>"${OUTPUT_FILE_NAME}" 2>&1
+    # Build dynamic header based on enabled fields
+    HEADER=""
+    
+    IsFieldEnabled "Org_Name" && HEADER="${HEADER}Org_Name,"
+    IsFieldEnabled "Repo_Name" && HEADER="${HEADER}Repo_Name,"
+    if [[ "$(isCompatibleGHESVersion)" -eq "1" ]]; then
+      IsFieldEnabled "Is_Empty" && HEADER="${HEADER}Is_Empty,"
+    fi
+    IsFieldEnabled "Last_Push" && HEADER="${HEADER}Last_Push,"
+    IsFieldEnabled "Last_Update" && HEADER="${HEADER}Last_Update,"
+    IsFieldEnabled "Latest_Commit_SHA" && HEADER="${HEADER}Latest_Commit_SHA,"
+    IsFieldEnabled "Latest_Committed_At" && HEADER="${HEADER}Latest_Committed_At,"
+    IsFieldEnabled "Latest_Commit_Branch" && HEADER="${HEADER}Latest_Commit_Branch,"
+    IsFieldEnabled "Default_Branch" && HEADER="${HEADER}Default_Branch,"
+    IsFieldEnabled "isFork" && HEADER="${HEADER}isFork,"
+    IsFieldEnabled "isArchived" && HEADER="${HEADER}isArchived,"
+    IsFieldEnabled "Repo_Size_mb" && HEADER="${HEADER}Repo_Size_mb,"
+    IsFieldEnabled "Record_Count" && HEADER="${HEADER}Record_Count,"
+    IsFieldEnabled "Collaborator_Count" && HEADER="${HEADER}Collaborator_Count,"
+    IsFieldEnabled "Protected_Branch_Count" && HEADER="${HEADER}Protected_Branch_Count,"
+    IsFieldEnabled "PR_Review_Count" && HEADER="${HEADER}PR_Review_Count,"
+    IsFieldEnabled "Milestone_Count" && HEADER="${HEADER}Milestone_Count,"
+    IsFieldEnabled "Issue_Count" && HEADER="${HEADER}Issue_Count,"
+    IsFieldEnabled "PR_Count" && HEADER="${HEADER}PR_Count,"
+    IsFieldEnabled "PR_Review_Comment_Count" && HEADER="${HEADER}PR_Review_Comment_Count,"
+    IsFieldEnabled "Commit_Comment_Count" && HEADER="${HEADER}Commit_Comment_Count,"
+    IsFieldEnabled "Issue_Comment_Count" && HEADER="${HEADER}Issue_Comment_Count,"
+    IsFieldEnabled "Issue_Event_Count" && HEADER="${HEADER}Issue_Event_Count,"
+    IsFieldEnabled "Release_Count" && HEADER="${HEADER}Release_Count,"
+    IsFieldEnabled "Project_Count" && HEADER="${HEADER}Project_Count,"
+    IsFieldEnabled "Branch_Count" && HEADER="${HEADER}Branch_Count,"
+    IsFieldEnabled "Tag_Count" && HEADER="${HEADER}Tag_Count,"
+    IsFieldEnabled "Discussion_Count" && HEADER="${HEADER}Discussion_Count,"
+    IsFieldEnabled "Has_Wiki" && HEADER="${HEADER}Has_Wiki,"
+    IsFieldEnabled "Full_URL" && HEADER="${HEADER}Full_URL,"
+    IsFieldEnabled "Migration_Issue" && HEADER="${HEADER}Migration_Issue,"
+    IsFieldEnabled "Created" && HEADER="${HEADER}Created,"
+
+    # Remove trailing comma
+    HEADER="${HEADER%,}"
+
+    echo "${HEADER}" >>"${OUTPUT_FILE_NAME}" 2>&1
 
     #######################
     # Load the error code #
@@ -1092,7 +1255,10 @@ ParseRepoDataBatch() {
   ########################
   # Write to temp file   #
   ########################
-  echo "${ORG_NAME},${REPO_NAME},${IS_EMPTY}${PUSHED_AT},${UPDATED_AT},${LATEST_COMMIT_SHA},${LATEST_COMMITTED_AT},${LATEST_COMMIT_BRANCH},${DEFAULT_BRANCH},${IS_FORK},${IS_ARCHIVED},${REPO_SIZE},${RECORD_CT},${COLLABORATOR_CT},${PROTECTED_BRANCH_CT},${PR_REVIEW_CT},${MILESTONE_CT},${ISSUE_CT},${PR_CT},${PR_REVIEW_COMMENT_CT},${COMMIT_COMMENT_CT},${ISSUE_COMMENT_CT},${ISSUE_EVENT_CT},${RELEASE_CT},${PROJECT_CT},${BRANCH_CT},${TAG_CT},${DISCUSSION_CT},${HAS_WIKI},${URL},${MIGRATION_ISSUE},${CREATED_AT}" > "${TEMP_OUTPUT_FILE}"
+  # Remove trailing comma from IS_EMPTY if present (it's handled by BuildOutputRow)
+  IS_EMPTY="${IS_EMPTY%,}"
+  OUTPUT_DATA=$(BuildOutputRow)
+  echo "${OUTPUT_DATA}" > "${TEMP_OUTPUT_FILE}"
   
   echo "Completed analysis for Repo: ${REPO_NAME}"
 }
@@ -1201,7 +1367,10 @@ ParseRepoData() {
   ########################
   # Write it to the file #
   ########################
-  echo "${ORG_NAME},${REPO_NAME},${IS_EMPTY}${PUSHED_AT},${UPDATED_AT},${LATEST_COMMIT_SHA},${LATEST_COMMITTED_AT},${LATEST_COMMIT_BRANCH},${DEFAULT_BRANCH},${IS_FORK},${IS_ARCHIVED},${REPO_SIZE},${RECORD_CT},${COLLABORATOR_CT},${PROTECTED_BRANCH_CT},${PR_REVIEW_CT},${MILESTONE_CT},${ISSUE_CT},${PR_CT},${PR_REVIEW_COMMENT_CT},${COMMIT_COMMENT_CT},${ISSUE_COMMENT_CT},${ISSUE_EVENT_CT},${RELEASE_CT},${PROJECT_CT},${BRANCH_CT},${TAG_CT},${DISCUSSION_CT},${HAS_WIKI},${URL},${MIGRATION_ISSUE},${CREATED_AT}" >>"${OUTPUT_FILE_NAME}"
+  # Remove trailing comma from IS_EMPTY if present (it's handled by BuildOutputRow)
+  IS_EMPTY="${IS_EMPTY%,}"
+  OUTPUT_DATA=$(BuildOutputRow)
+  echo "${OUTPUT_DATA}" >>"${OUTPUT_FILE_NAME}"
 
   #######################
   # Load the error code #
@@ -1869,6 +2038,11 @@ ValidateInput() {
 ##########
 Header
 
+###########################
+# Load Field Config       #
+###########################
+LoadFieldConfig
+
 #########################
 # Validate JQ installed #
 #########################
