Manage paged results from gitea API.

generateRepoList.sh

@@ -1,9 +1,25 @@
 #!/usr/bin/env bash
+organization="gCubeSystem"
+pageCounter=1
 timestamp=$(date +"%Y-%m-%d_%H-%M-%S")
-curl -X GET "https://code-repo.d4science.org/api/v1/orgs/gCubeSystem/repos" -H  "accept: application/json" | python -mjson.tool > allp.json
+empty="false"
+echo "" > allp.json
+while [ $empty = "false" ]
+do
+  page=$(curl -X GET "https://code-repo.d4science.org/api/v1/orgs/${organization}/repos?page=${pageCounter}&limit=100000" -H "accept: application/json")
+  echo "Length ${#page}"
+  #break the loop when we get an empty json response ([])
+  if [ ${#page} -le 2 ]; then
+    empty="true"
+  else
+    echo $page | python -mjson.tool >> allp.json
+    ((pageCounter++))
+  fi
+done
+
 grep "\"name\": " allp.json > all_filtered.json
 sed "s/^[ \t]*\"name\": \"//g" all_filtered.json > all_filtered1.txt
 sed 's/\",//g' all_filtered1.txt > all_filtered2.txt
 sort all_filtered2.txt > all_sorted.txt
-mv all_sorted.txt gCubeSystem_all_sorted_$timestamp.txt
+mv all_sorted.txt ${organization}_all_sorted_$timestamp.txt
 rm allp.json all_filtered.json all_filtered1.txt all_filtered2.txt