package com.paxata.sample; import org.apache.commons.codec.binary.Base64; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.*; import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.StringBody; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; import org.json.JSONArray; import org.json.JSONObject; import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Set; public class Main { public static void main(String[] args) { System.out.println("Importing into Paxata through query..."); JSONObject qryDS = importQuery("601f097a63084330bf2f67ae33ed6357", "select * from devdb.1118"); System.out.println("Importing a table into Paxata..."); JSONObject tblDSID = importTable("601f097a63084330bf2f67ae33ed6357", "/devdb/1118"); System.out.println("Create a project..."); JSONObject prj = createProject("TestProject1", "Project created through REST API."); System.out.println("Ingest base dataset in project..."); ingestBaseDS(prj, qryDS); System.out.println("Create a new version of the query dataset..."); createVersion("601f097a63084330bf2f67ae33ed6357", "select * from devdb.1118", qryDS.getString("dataFileId")); } private static JSONObject importQuery(String dsId, String qry) { HashMap opts = new HashMap<>(); JSONObject ds; opts.put("query", qry); opts.put("name", "importedViaQuery"); HttpResponse resp = executeRequest(opts, "POST", "/rest/datasource/imports/", "/" + dsId, null); System.out.println(resp.getStatusLine().getStatusCode() + " - " + resp.getStatusLine().getReasonPhrase()); try { ds = new JSONObject(EntityUtils.toString(resp.getEntity(), "UTF-8")); return ds; } catch (IOException e) { e.printStackTrace(); } return null; } private static JSONObject importTable(String dsId, String table) { HashMap opts = new HashMap<>(); JSONObject ds; opts.put("itemPath", table); opts.put("name", table); HttpResponse resp = executeRequest(opts, "POST", "/rest/datasource/imports/", "/" + dsId, null); System.out.println(resp.getStatusLine().getStatusCode() + " - " + resp.getStatusLine().getReasonPhrase()); try { ds = new JSONObject(EntityUtils.toString(resp.getEntity(), "UTF-8")); return ds; } catch (IOException e) { e.printStackTrace(); } return null; } private static JSONObject createProject(String name, String desc) { HashMap opts = new HashMap<>(); JSONObject proj; opts.put("name", name); opts.put("description", desc); HttpResponse resp = executeRequest(opts, "POST", "/rest/projects", "", null); System.out.println(resp.getStatusLine().getStatusCode() + " - " + resp.getStatusLine().getReasonPhrase()); try { proj = new JSONObject(EntityUtils.toString(resp.getEntity(), "UTF-8"));//.getString("projectId"); return proj; } catch (IOException e) { e.printStackTrace(); } return null; } private static void ingestBaseDS(JSONObject project, JSONObject baseMetadata) { HashMap opts = new HashMap<>(); JSONObject jsonObj; JSONArray cols = new JSONArray(); for (int c = 0, s = baseMetadata.getJSONArray("schema").length(); c < s; c++) { JSONObject currentSchema = baseMetadata.getJSONArray("schema").getJSONObject(c); JSONObject col = new JSONObject(); col.put("columnDisplayName", currentSchema.get("name")); col.put("columnName", currentSchema.get("name")); col.put("columnType", currentSchema.get("type")); col.put("hidden", false); cols.put(col); } HttpResponse resp = executeRequest(null, "get", "/rest/scripts", "?version=-1&projectId=" + project.getString("projectId"), null); if (resp.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { System.out.println(resp.getStatusLine().getStatusCode() + " - " + resp.getStatusLine().getReasonPhrase()); } try { jsonObj = new JSONObject(EntityUtils.toString(resp.getEntity(), "UTF-8")); JSONObject step = jsonObj.getJSONArray("steps").getJSONObject(0); JSONObject importStep = step.getJSONObject("importStep"); importStep.put("libraryId", baseMetadata.getString("dataFileId")); importStep.put("libraryVersion", "" + baseMetadata.getInt("version")); importStep.put("columns", cols); importStep.put("libraryIdWithVersion", baseMetadata.getString("dataFileId") + "_" + baseMetadata.getInt("version")); JSONArray scripts = new JSONArray(); scripts.put(step); jsonObj.put("steps", scripts); opts.put("force", "true"); opts.put("update", "script"); opts.put("projectId", project.getString("projectId")); opts.put("script", jsonObj.toString()); resp = executeRequest(opts, "PUT", "/rest/scripts", "", null); System.out.println(resp.getStatusLine().getStatusCode() + " - " + resp.getStatusLine().getReasonPhrase()); } catch (IOException e) { e.printStackTrace(); } } private static void createVersion(String dsId, String qry, String dfId) { HashMap opts = new HashMap<>(); JSONObject ds; opts.put("query", qry); opts.put("name", "importedViaQuery"); HttpResponse resp = executeRequest(opts, "PUT", "/rest/datasource/imports/", "/" + dsId + "/" + dfId, null); System.out.println(resp.getStatusLine().getStatusCode() + " - " + resp.getStatusLine().getReasonPhrase()); } private static HttpResponse executeRequest(HashMap reqOpt, String verb, String endpoint, String opt, File f) { CredentialsProvider provider = new BasicCredentialsProvider(); HttpClient httpclient = HttpClients.custom().setSSLHostnameVerifier(new NoopHostnameVerifier()).setDefaultCredentialsProvider(provider).build(); HttpResponse response = null; HttpRequestBase request = null; String url = endpoint + opt; String paxToken = ":c151fbfa18c94b95b0dde2119df1a8fb"; String authToken = new Base64().encodeToString(paxToken.getBytes()); String paxURL = "https://dataprep.paxata.com"; if (verb.equalsIgnoreCase("GET")) { request = new HttpGet(paxURL + url); } else if (verb.equalsIgnoreCase("PUT")) { request = new HttpPut(paxURL + url); } else if (verb.equalsIgnoreCase("POST")) { request = new HttpPost(paxURL + url); } else if (verb.equalsIgnoreCase("DELETE")) { request = new HttpDelete(paxURL + url); } if (request != null) { request.addHeader("Authorization", "Basic " + authToken); } MultipartEntityBuilder builder = MultipartEntityBuilder.create(); if (reqOpt != null) { Set> entrySet = reqOpt.entrySet(); for (Map.Entry entry : entrySet) { String key = entry.getKey(); String value = entry.getValue(); builder.addPart(key, new StringBody(value, ContentType.MULTIPART_FORM_DATA)); } } if (f != null) { String importFileName = f.getName(); builder.addBinaryBody("data", f, ContentType.APPLICATION_OCTET_STREAM, importFileName); } HttpEntity multipart = builder.build(); if (request instanceof HttpPut) ((HttpPut) request).setEntity(multipart); if (request instanceof HttpPost) ((HttpPost) request).setEntity(multipart); try { response = httpclient.execute(request); } catch (IOException e) { e.printStackTrace(); } return response; } }