Merge pull request #151 from cloudsufi/feature/proxy-support

[PLUGIN-1959] Add proxy support for ServiceNow plugins

Author: vikasrathee-cs

docs/ServiceNow-batchsink.md

@@ -24,6 +24,15 @@ You also can use the macro function ${conn(connection-name)}.
 
 **Password**: The password for ServiceNow Instance.
 
+**Proxy URL**: Proxy URL through which all the ServiceNow API calls are routed. For example,
+`http://proxy.example.com:8080`. If no scheme is specified (for example, `proxy.example.com:8080`),
+it defaults to `http`. Specify `https://` explicitly
+when using an HTTPS proxy. Leave it empty to connect to ServiceNow directly.
+
+**Proxy Username**: The username to authenticate with the proxy server, if the proxy requires authentication.
+
+**Proxy Password**: The password to authenticate with the proxy server, if the proxy requires authentication.
+
 **Reference Name**: Name used to uniquely identify this source for lineage, annotating metadata, etc.
 
 **Table Name**: The name of the ServiceNow table into which data is to be pushed.

docs/ServiceNow-batchsource.md

@@ -27,6 +27,15 @@ You also can use the macro function ${conn(connection-name)}.
 
 **Password**: The password for ServiceNow Instance.
 
+**Proxy URL**: Proxy URL through which all the ServiceNow API calls are routed. For example,
+`http://proxy.example.com:8080`. If no scheme is specified (for example, `proxy.example.com:8080`),
+it defaults to `http`. Specify `https://` explicitly
+when using an HTTPS proxy. Leave it empty to connect to ServiceNow directly.
+
+**Proxy Username**: The username to authenticate with the proxy server, if the proxy requires authentication.
+
+**Proxy Password**: The password to authenticate with the proxy server, if the proxy requires authentication.
+
 **Reference Name**: Name used to uniquely identify this source for lineage, annotating metadata, etc.
 
 **Mode**: Mode of query. The mode can be one of two values: 

docs/ServiceNow-connector.md

@@ -20,6 +20,15 @@ Properties
 
 **Password**: The password for ServiceNow Instance.
 
+**Proxy URL**: Proxy URL through which all the ServiceNow API calls are routed. For example,
+`http://proxy.example.com:8080`. If no scheme is specified (for example, `proxy.example.com:8080`),
+it defaults to `http`. Specify `https://` explicitly
+when using an HTTPS proxy. Leave it empty to connect to ServiceNow directly.
+
+**Proxy Username**: The username to authenticate with the proxy server, if the proxy requires authentication.
+
+**Proxy Password**: The password to authenticate with the proxy server, if the proxy requires authentication.
+
 
 Path of the connection
 ----------------------

docs/ServiceNowMultiSource-batchsource.md

@@ -22,6 +22,15 @@ Properties
 
 **Password**: The password for ServiceNow Instance.
 
+**Proxy URL**: Proxy URL through which all the ServiceNow API calls are routed. For example,
+`http://proxy.example.com:8080`. If no scheme is specified (for example, `proxy.example.com:8080`),
+it defaults to `http`. Specify `https://` explicitly
+when using an HTTPS proxy. Leave it empty to connect to ServiceNow directly.
+
+**Proxy Username**: The username to authenticate with the proxy server, if the proxy requires authentication.
+
+**Proxy Password**: The password to authenticate with the proxy server, if the proxy requires authentication.
+
 **Reference Name**: Name used to uniquely identify this source for lineage, annotating metadata, etc.
 
 **Table Names**: The name of the ServiceNow table(s) from which data to be fetched.

pom.xml

@@ -105,6 +105,10 @@
       <version>${hadoop.version}</version>
       <scope>provided</scope>
       <exclusions>
+        <exclusion>
+          <groupId>commons-codec</groupId>
+          <artifactId>commons-codec</artifactId>
+        </exclusion>
         <exclusion>
           <groupId>commons-logging</groupId>
           <artifactId>commons-logging</artifactId>

src/e2e-test/resources/errorMessage.properties

@@ -5,7 +5,7 @@ validationSuccessMessage=No errors found.
 invalid.property.tablename=ServiceNow API returned an unexpected result or the specified table may not exist. Cause: Http call to ServiceNow instance returned status code 400. Ensure specified table exists in the datasource.
 invalid.property.startdate=Invalid format for Start date. Correct Format: yyyy-MM-dd
 invalid.property.enddate=Invalid format for End date. Correct Format: yyyy-MM-dd
-invalid.property.credentials=Unable to connect to ServiceNow Instance. Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password are correct.
+invalid.property.credentials=Unable to connect to ServiceNow Instance. Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password and Proxy Properties are correct.
 
 #Logs error message
 invalid.tablename.logsmessage=ServiceNow API returned an unexpected result or the specified table may not exist.

src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java

@@ -89,13 +89,16 @@ public void validateServiceNowConnection(FailureCollector collector) {
       restApi.getAccessToken();
     } catch (Exception e) {
       collector.addFailure("Unable to connect to ServiceNow Instance.",
-                           "Ensure properties like Client ID, Client Secret, API Endpoint, User Name, Password " +
-                             "are correct.")
+                           "Ensure properties like Client ID, Client Secret, API Endpoint, User Name, " +
+                                   "Password and Proxy Properties are correct.")
         .withConfigProperty(ServiceNowConstants.PROPERTY_CLIENT_ID)
         .withConfigProperty(ServiceNowConstants.PROPERTY_CLIENT_SECRET)
         .withConfigProperty(ServiceNowConstants.PROPERTY_API_ENDPOINT)
         .withConfigProperty(ServiceNowConstants.PROPERTY_USER)
         .withConfigProperty(ServiceNowConstants.PROPERTY_PASSWORD)
+        .withConfigProperty(ServiceNowConstants.PROPERTY_PROXY_URL)
+        .withConfigProperty(ServiceNowConstants.PROPERTY_PROXY_USERNAME)
+        .withConfigProperty(ServiceNowConstants.PROPERTY_PROXY_PASSWORD)
         .withStacktrace(e.getStackTrace());
     }
   }
@@ -108,7 +111,10 @@ public boolean shouldConnect() {
       !containsMacro(ServiceNowConstants.PROPERTY_CLIENT_SECRET) &&
       !containsMacro(ServiceNowConstants.PROPERTY_API_ENDPOINT) &&
       !containsMacro(ServiceNowConstants.PROPERTY_USER) &&
-      !containsMacro(ServiceNowConstants.PROPERTY_PASSWORD);
+      !containsMacro(ServiceNowConstants.PROPERTY_PASSWORD) &&
+      !containsMacro(ServiceNowConstants.PROPERTY_PROXY_URL)  &&
+      !containsMacro(ServiceNowConstants.PROPERTY_PROXY_USERNAME)  &&
+      !containsMacro(ServiceNowConstants.PROPERTY_PROXY_PASSWORD);
   }
 
   public boolean shouldGetSchema() {

src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java

@@ -49,7 +49,6 @@
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
 import org.apache.oltu.oauth2.common.exception.OAuthProblemException;
 import org.apache.oltu.oauth2.common.exception.OAuthSystemException;
 import org.slf4j.Logger;
@@ -86,6 +85,7 @@ public class ServiceNowTableAPIClientImpl extends RestAPIClient {
   public static JsonArray serviceNowJsonResultArray;
 
   public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf, Boolean useConnection) {
+    super(conf.getProxyUrl(), conf.getProxyUsername(), conf.getProxyPassword());
     this.conf = conf;
     this.schemaType = getSchemaTypeBasedOnUseConnection(useConnection);
   }
@@ -561,7 +561,7 @@ public void deleteRecordFromServiceNowTable(String tableName, String sysId)
     deleteRequest.setHeader("Authorization", "Bearer " + accessToken);
     deleteRequest.setHeader("Accept", "application/json");
 
-    try (CloseableHttpClient httpClient = HttpClients.createDefault();
+    try (CloseableHttpClient httpClient = getHttpClientBuilder().build();
          CloseableHttpResponse response = httpClient.execute(deleteRequest)) {
 
       int statusCode = response.getStatusLine().getStatusCode();

src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java

@@ -23,6 +23,7 @@
 import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl;
 import io.cdap.plugin.servicenow.util.ServiceNowConstants;
 import io.cdap.plugin.servicenow.util.Util;
+import org.apache.http.HttpHost;
 
 import javax.annotation.Nullable;
 
@@ -62,13 +63,41 @@ public class ServiceNowConnectorConfig extends PluginConfig {
   @Description("The password for ServiceNow Instance.")
   private final String password;
 
+  @Name(ServiceNowConstants.PROPERTY_PROXY_URL)
+  @Macro
+  @Nullable
+  @Description("Proxy URL through which all the ServiceNow API calls are routed. " +
+    "Must contain a protocol, address and port. For example, http://proxy.example.com:8080")
+  private final String proxyUrl;
+
+  @Name(ServiceNowConstants.PROPERTY_PROXY_USERNAME)
+  @Macro
+  @Nullable
+  @Description("The username to authenticate with the proxy server, if the proxy requires authentication.")
+  private final String proxyUsername;
+
+  @Name(ServiceNowConstants.PROPERTY_PROXY_PASSWORD)
+  @Macro
+  @Nullable
+  @Description("The password to authenticate with the proxy server, if the proxy requires authentication.")
+  private final String proxyPassword;
+
   public ServiceNowConnectorConfig(String clientId, String clientSecret, String restApiEndpoint,
                                    String user, String password) {
+    this(clientId, clientSecret, restApiEndpoint, user, password, null, null, null);
+  }
+
+  public ServiceNowConnectorConfig(String clientId, String clientSecret, String restApiEndpoint,
+                                   String user, String password, String proxyUrl, String proxyUsername,
+                                   String proxyPassword) {
     this.clientId = clientId;
     this.clientSecret = clientSecret;
     this.restApiEndpoint = restApiEndpoint;
     this.user = user;
     this.password = password;
+    this.proxyUrl = proxyUrl;
+    this.proxyUsername = proxyUsername;
+    this.proxyPassword = proxyPassword;
   }
 
   public String getClientId() {
@@ -91,6 +120,28 @@ public String getPassword() {
     return password;
   }
 
+  @Nullable
+  public String getProxyUrl() {
+    return proxyUrl;
+  }
+
+  @Nullable
+  public String getProxyUsername() {
+    return proxyUsername;
+  }
+
+  @Nullable
+  public String getProxyPassword() {
+    return proxyPassword;
+  }
+
+  /**
+   * Returns true if the proxy properties do not contain macros and a proxy URL has been configured.
+   */
+  public boolean hasProxyConfigured() {
+    return !containsMacro(ServiceNowConstants.PROPERTY_PROXY_URL) && !Util.isNullOrEmpty(proxyUrl);
+  }
+
 
   /**
    * validates all the fields which are mandatory for the connection.
@@ -120,6 +171,43 @@ public void validateCredentialsFields(FailureCollector collector) {
       collector.addFailure("Password must be specified.", null)
         .withConfigProperty(ServiceNowConstants.PROPERTY_PASSWORD);
     }
+
+    validateProxyFields(collector);
+  }
+
+  /**
+   * Validates the optional proxy fields. The proxy URL must be a valid host with a protocol and port, and a proxy
+   * username and password must be provided together.
+   */
+  public void validateProxyFields(FailureCollector collector) {
+    if (!containsMacro(ServiceNowConstants.PROPERTY_PROXY_URL) && !Util.isNullOrEmpty(proxyUrl)) {
+      try {
+        HttpHost host = HttpHost.create(proxyUrl);
+        if (host.getPort() == -1) {
+          throw new IllegalArgumentException();
+        }
+      } catch (IllegalArgumentException e) {
+        collector.addFailure("Proxy URL is not valid.",
+                             "Ensure the Proxy URL contains a protocol, address and port. " +
+                               "For example, http://proxy.example.com:8080.")
+          .withConfigProperty(ServiceNowConstants.PROPERTY_PROXY_URL);
+      }
+    }
+
+    if (containsMacro(ServiceNowConstants.PROPERTY_PROXY_USERNAME)
+      || containsMacro(ServiceNowConstants.PROPERTY_PROXY_PASSWORD)) {
+      return;
+    }
+
+    boolean hasUsername = !Util.isNullOrEmpty(proxyUsername);
+    boolean hasPassword = !Util.isNullOrEmpty(proxyPassword);
+    if (hasUsername && !hasPassword) {
+      collector.addFailure("Proxy password must be specified when a proxy username is provided.", null)
+        .withConfigProperty(ServiceNowConstants.PROPERTY_PROXY_PASSWORD);
+    } else if (!hasUsername && hasPassword) {
+      collector.addFailure("Proxy username must be specified when a proxy password is provided.", null)
+        .withConfigProperty(ServiceNowConstants.PROPERTY_PROXY_USERNAME);
+    }
   }
 
   public void validateConnection(FailureCollector collector) {

src/main/java/io/cdap/plugin/servicenow/restapi/ProxyOltuHttpClient.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright © 2026 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.plugin.servicenow.restapi;
+
+import org.apache.http.Header;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.apache.oltu.oauth2.client.HttpClient;
+import org.apache.oltu.oauth2.client.request.OAuthClientRequest;
+import org.apache.oltu.oauth2.client.response.OAuthClientResponse;
+import org.apache.oltu.oauth2.client.response.OAuthClientResponseFactory;
+import org.apache.oltu.oauth2.common.OAuth;
+import org.apache.oltu.oauth2.common.exception.OAuthProblemException;
+import org.apache.oltu.oauth2.common.exception.OAuthSystemException;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * An implementation of Oltu's {@link HttpClient} backed by Apache {@link CloseableHttpClient}. It is used so that the
+ * OAuth token request is executed through the same proxy-aware HTTP client as the rest of the ServiceNow API calls.
+ */
+public class ProxyOltuHttpClient implements HttpClient {
+
+  private final HttpClientBuilder httpClientBuilder;
+
+  public ProxyOltuHttpClient(HttpClientBuilder httpClientBuilder) {
+    this.httpClientBuilder = httpClientBuilder;
+  }
+
+  @Override
+  public <T extends OAuthClientResponse> T execute(OAuthClientRequest request, Map<String, String> headers,
+                                                   String requestMethod, Class<T> responseClass)
+    throws OAuthSystemException, OAuthProblemException {
+
+    HttpRequestBase httpRequest;
+    if (OAuth.HttpMethod.POST.equalsIgnoreCase(requestMethod) || OAuth.HttpMethod.PUT.equalsIgnoreCase(requestMethod)) {
+      HttpPost httpPost = new HttpPost(request.getLocationUri());
+      if (request.getBody() != null) {
+        httpPost.setEntity(new StringEntity(request.getBody(), StandardCharsets.UTF_8));
+      }
+      httpRequest = httpPost;
+    } else {
+      httpRequest = new HttpGet(request.getLocationUri());
+    }
+
+    if (request.getHeaders() != null) {
+      request.getHeaders().forEach(httpRequest::setHeader);
+    }
+    if (headers != null) {
+      headers.forEach(httpRequest::setHeader);
+    }
+    if (httpRequest.getFirstHeader(OAuth.HeaderType.CONTENT_TYPE) == null) {
+      httpRequest.setHeader(OAuth.HeaderType.CONTENT_TYPE, OAuth.ContentType.URL_ENCODED);
+    }
+
+    try (CloseableHttpClient httpClient = httpClientBuilder.build();
+         CloseableHttpResponse response = httpClient.execute(httpRequest)) {
+      int responseCode = response.getStatusLine().getStatusCode();
+      String contentType = response.getEntity() != null && response.getEntity().getContentType() != null
+        ? response.getEntity().getContentType().getValue() : null;
+      String body = response.getEntity() != null ? EntityUtils.toString(response.getEntity()) : "";
+      return OAuthClientResponseFactory.createCustomResponse(body, contentType, responseCode,
+                                                             extractHeaders(response), responseClass);
+    } catch (IOException e) {
+      throw new OAuthSystemException(e);
+    }
+  }
+
+  @Override
+  public void shutdown() {
+    // No persistent resources are held; a fresh client is created per request and closed in execute().
+  }
+
+  private Map<String, List<String>> extractHeaders(CloseableHttpResponse response) {
+    Map<String, List<String>> responseHeaders = new java.util.HashMap<>();
+    for (Header header : response.getAllHeaders()) {
+      List<String> values = responseHeaders.computeIfAbsent(header.getName(), k -> new ArrayList<>());
+      values.add(header.getValue());
+    }
+    return Collections.unmodifiableMap(responseHeaders);
+  }
+}