Merge branch 'main' into bug6240

Author: axeluhl

configuration/environments_scripts/sailing_server/files/usr/local/bin/refreshInstance.sh

@@ -153,11 +153,11 @@ install_environment ()
 load_from_release_file ()
 {
     if [[ ${INSTALL_FROM_RELEASE} == "" ]]; then
-        GITHUB_RELEASE=$( curl -L "https://api.github.com/repos/SAP/sailing-analytics/releases?per_page=100" 2>/dev/null | jq -r 'sort_by(.created_at) | reverse | map(select(.name | startswith("main-")))[0].assets[] | select(.content_type=="application/x-tar")' )
+        GITHUB_RELEASE=$( curl -L "https://api.github.com/repos/SAP/sailing-analytics/releases?per_page=100" 2>/dev/null | jq -r 'map(select(.name | startswith("main-")) | .assets[] | select(.content_type=="application/x-tar")) | sort_by(.created_at) | reverse | first' )
         INSTALL_FROM_RELEASE=$( echo "${GITHUB_RELEASE}" | jq -r '.name' | sed -e 's/\.tar\.gz$//' )
         echo "You didn't provide a release. Defaulting to latest main branch build ${INSTALL_FROM_RELEASE}"
     else
-        GITHUB_RELEASE=$( curl -L "https://api.github.com/repos/SAP/sailing-analytics/releases?per_page=100" 2>/dev/null | jq -r 'sort_by(.created_at) | reverse | map(select(.name=="'${INSTALL_FROM_RELEASE}'"))[0].assets[] | select(.content_type=="application/x-tar")' )
+        GITHUB_RELEASE=$( curl -L "https://api.github.com/repos/SAP/sailing-analytics/releases?per_page=100" 2>/dev/null | jq -r 'map(select(.name=="'${INSTALL_FROM_RELEASE}'") | .assets[] | select(.content_type=="application/x-tar")) | sort_by(.created_at) | reverse | first' )
     fi
     if which mail; then
         if [ -n "${BUILD_COMPLETE_NOTIFY}" ]; then

java/com.sap.sailing.server/SailingServer (No Proxy).launch

@@ -14,6 +14,9 @@
     <setAttribute key="deselected_workspace_bundles"/>
     <booleanAttribute key="includeOptional" value="false"/>
     <booleanAttribute key="org.eclipse.debug.core.ATTR_FORCE_SYSTEM_CONSOLE_ENCODING" value="false"/>
+    <mapAttribute key="org.eclipse.debug.core.environmentVariables">
+        <mapEntry key="GITHUB_TOKEN" value="${GITHUB_TOKEN}"/>
+    </mapAttribute>
     <stringAttribute key="org.eclipse.debug.core.source_locator_id" value="org.eclipse.pde.ui.launcher.PDESourceLookupDirector"/>
     <stringAttribute key="org.eclipse.debug.core.source_locator_memento" value="&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot; standalone=&quot;no&quot;?&gt;&#13;&#10;&lt;sourceLookupDirector&gt;&#13;&#10;    &lt;sourceContainers duplicates=&quot;false&quot;&gt;&#13;&#10;        &lt;container memento=&quot;&amp;lt;?xml version=&amp;quot;1.0&amp;quot; encoding=&amp;quot;UTF-8&amp;quot; standalone=&amp;quot;no&amp;quot;?&amp;gt;&amp;#13;&amp;#10;&amp;lt;directory nest=&amp;quot;false&amp;quot; path=&amp;quot;C:\Users\georg\Documents\Business\Kunden\OIO\SAPsailing\jetty.project\apache-jsp\src\main\java&amp;quot;/&amp;gt;&amp;#13;&amp;#10;&quot; typeId=&quot;org.eclipse.debug.core.containerType.directory&quot;/&gt;&#13;&#10;        &lt;container memento=&quot;&amp;lt;?xml version=&amp;quot;1.0&amp;quot; encoding=&amp;quot;UTF-8&amp;quot; standalone=&amp;quot;no&amp;quot;?&amp;gt;&amp;#13;&amp;#10;&amp;lt;classpathContainer path=&amp;quot;com.google.gwt.eclipse.core.GWT_CONTAINER&amp;quot;/&amp;gt;&amp;#13;&amp;#10;&quot; typeId=&quot;org.eclipse.jdt.launching.sourceContainer.classpathContainer&quot;/&gt;&#13;&#10;        &lt;container memento=&quot;&amp;lt;?xml version=&amp;quot;1.0&amp;quot; encoding=&amp;quot;UTF-8&amp;quot; standalone=&amp;quot;no&amp;quot;?&amp;gt;&amp;#13;&amp;#10;&amp;lt;default/&amp;gt;&amp;#13;&amp;#10;&quot; typeId=&quot;org.eclipse.debug.core.containerType.default&quot;/&gt;&#13;&#10;    &lt;/sourceContainers&gt;&#13;&#10;&lt;/sourceLookupDirector&gt;&#13;&#10;"/>
     <listAttribute key="org.eclipse.debug.ui.favoriteGroups">

java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/GithubReleasesRepository.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLConnection;
@@ -24,6 +25,7 @@
 
 import com.sap.sse.common.Duration;
 import com.sap.sse.common.TimePoint;
+import com.sap.sse.common.Util;
 import com.sap.sse.common.Util.Pair;
 import com.sap.sse.landscape.Release;
 import com.sap.sse.landscape.ReleaseRepository;
@@ -153,8 +155,8 @@ private ReleaseIterator() throws MalformedURLException, IOException, ParseExcept
                 } else {
                     logger.fine(()->"Need to fetch page with newest releases because last request was at "+
                             (lastFetchOfNewestReleases==null?"<never>":lastFetchOfNewestReleases));
-                    lastFetchOfNewestReleases = now;
                     fillCacheWithNewestReleases();
+                    lastFetchOfNewestReleases = now;
                 }
                 cachedReleasesIterator = releasesByPublishingTimePoint.descendingMap().values().iterator();
             }
@@ -215,14 +217,36 @@ public GithubReleasesRepository(String owner, String repositoryName, String defa
      * cache when this method is invoked.
      * <p>
      * 
+     * If the {@code GITHUB_TOKEN} environment variable is set, it is used as a bearer token forthe Github requests,
+     * resulting in a higher rate limit. If no token is set, or if using the token results in a 401 response code
+     * (authentication failed), an unauthenticated request is tried instead.
+     * <p>
+     * 
      * The method makes no changes to the cache or any other state of this instance.
      * 
      * @return the link to the next page in the returned pair's {@link Pair#getA() A component}, and the sequence of
      *         releases loaded from that page with their publishing time points, ordered from newest to oldest.
      */
     private synchronized Pair<String, Iterable<Pair<TimePoint, GithubRelease>>> getReleasesFromPage(String pageURL) throws IOException, ParseException {
         logger.info("Requesting releases page "+pageURL);
-        final URLConnection connection = HttpUrlConnectionHelper.redirectConnection(new URL(pageURL));
+        final URLConnection connection;
+        final String githubToken = System.getenv("GITHUB_TOKEN");
+        if (Util.hasLength(githubToken)) {
+            final HttpURLConnection authenticatedConnectionAttempt = (HttpURLConnection) HttpUrlConnectionHelper
+                    .redirectConnectionWithBearerToken(new URL(pageURL), githubToken);
+            if (authenticatedConnectionAttempt.getResponseCode() == 401) {
+                try {
+                    authenticatedConnectionAttempt.disconnect();
+                } catch (Exception e) {
+                    logger.warning("Couldn't disconnect from Github: "+e.getMessage());
+                }
+                connection = HttpUrlConnectionHelper.redirectConnection(new URL(pageURL));
+            } else {
+                connection = authenticatedConnectionAttempt;
+            }
+        } else {
+            connection = HttpUrlConnectionHelper.redirectConnection(new URL(pageURL));
+        }
         final InputStream index = (InputStream) connection.getContent();
         final String xRatelimitRemaining = connection.getHeaderField("x-ratelimit-remaining");
         logger.fine(()->""+xRatelimitRemaining+" requests left in this hour");

java/target/compareServers

@@ -68,15 +68,15 @@ if [ "${useRestApi}" = "1" ]; then
     shift
   done
   CURL_OUTPUT=$( curl -sL --dump-header "${HEADERS}" -H 'Content-Type: multipart/form-data' "${NEWSERVER}/sailingserver/api/v1/compareservers" -d "server2=${OLDSERVER}" ${AUTH} ${LEADERBOARD_GROUPS} 2>${WGET_ERR} )
-  echo "${CURL_OUTPUT}" | jq
+  echo "${CURL_OUTPUT}"
   HTTP_STATUS=$( cat "${HEADERS}" | grep "^HTTP\/[0-9.]* [0-9]\{3\}" | tail -1 | sed -e 's/^HTTP\/[0-9.]* \([0-9]\{3\}\).*$/\1/' )
   rm -f "${HEADERS}"
   MESSAGE_SUFFIX=". HTTP_STATUS was ${HTTP_STATUS}."
   if [[ "${HTTP_STATUS}" =~ 2.. ]]; then
-    echo "Success${MESSAGE_SUFFIX}"
+    echo "Success${MESSAGE_SUFFIX}" >&2
     exit 0
   else
-    echo "Failed${MESSAGE_SUFFIX}"
+    echo "Failed${MESSAGE_SUFFIX}" >&2
     exit ${HTTP_STATUS}
   fi
 else

java/target/env-default-rules.sh

@@ -140,3 +140,4 @@ fi
 if [ -n "${GEONAMES_ORG_USERNAMES}" ]; then
   ADDITIONAL_JAVA_ARGS="${ADDITIONAL_JAVA_ARGS} -Dgeonames.org.usernames=${GEONAMES_ORG_USERNAMES}"
 fi
+export GITHUB_TOKEN

wiki/howto/onboarding.md

@@ -115,6 +115,7 @@ Go to Window ⇒ Preferences and change the following two settings:
 - For YouTube API access, provide an Eclipse variable ``YOUTUBE_API_KEY``. You can use your own Google developer account to create such a key; see [here](https://console.cloud.google.com).
 - In case you need to test with the payment provider (ChargeBee) active, you need to provide Eclipse variables (Run/Rebug ⇒ String Substitution) ``CHARGEBEE_SITE`` and ``CHARGEBEE_API_KEY`` where the recommendation is to use ``sailytics-test`` for the site and a corresponding test API key; there is one launch configuration running against the live ChargeBee site, but its use is not recommended for obvious reasons. For it, define the variables ``CHARGEBEE_SITE_LIVE`` and ``CHARGEBEE_API_KEY_LIVE``, respectively.
 - To successfully obtain polar and wind estimation data from ``sapsailing.com``, you need to define the Eclipse variables (Run/Debug ⇒ String Substitution) ``POLAR_DATA_BEARER_TOKEN`` and ``WIND_ESTIMATION_MODEL_BEARER_TOKEN`` that need to authenticate a user with permissions to read the respective models.
+- Define a ``GITHUB_TOKEN`` variable; it may be empty (but has to be generally present), or you can use it to set a personal access token (PAT) which will then be used, in particular, for requesting application releases from GitHub.
 - To work with the wind estimation model training launch configuration (e.g., SimpleModelsTrainingPart1.launch), define an Eclipse variable (Run/Debug ⇒ String Substitution) ``SAPSAILING_BEARER_TOKEN`` which needs to authenticate a user eligible to read all maneuver data from sapsailing.com
 - In "GWT ⇒ Errors/Warnings" set "Missing SDK" to "Ignore" 
 - In "GWT ⇒ GWT Settings ⇒ Add..." add the GWT SDK you downloaded and unpacked earlier