diff --git a/addOns/postman/src/main/java/org/zaproxy/addon/postman/PostmanParser.java b/addOns/postman/src/main/java/org/zaproxy/addon/postman/PostmanParser.java index 18e211e9fd3..1a89a02d61f 100644 --- a/addOns/postman/src/main/java/org/zaproxy/addon/postman/PostmanParser.java +++ b/addOns/postman/src/main/java/org/zaproxy/addon/postman/PostmanParser.java @@ -24,12 +24,33 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; import org.apache.commons.httpclient.URI; +import org.apache.commons.httpclient.URIException; import org.apache.commons.io.FileUtils; import org.parosproxy.paros.Constant; +import org.parosproxy.paros.network.HttpHeader; +import org.parosproxy.paros.network.HttpMalformedHeaderException; +import org.parosproxy.paros.network.HttpMessage; import org.parosproxy.paros.network.HttpSender; +import org.zaproxy.addon.postman.models.AbstractItem; +import org.zaproxy.addon.postman.models.Body; +import org.zaproxy.addon.postman.models.Body.FormData; +import org.zaproxy.addon.postman.models.Body.GraphQl; +import org.zaproxy.addon.postman.models.Item; +import org.zaproxy.addon.postman.models.ItemGroup; +import org.zaproxy.addon.postman.models.KeyValueData; import org.zaproxy.addon.postman.models.PostmanCollection; +import org.zaproxy.addon.postman.models.Request; +import org.zaproxy.addon.postman.models.Request.Url; public class PostmanParser { @@ -75,7 +96,10 @@ public void importFromUrl(final String url) throws IllegalArgumentException, IOE public void importCollection(String collection) throws JsonProcessingException { PostmanCollection postmanCollection = parse(collection); - // TODO: Extract list of HttpMessage from PostmanCollection and send requests + List httpMessages = new ArrayList<>(); + extractHttpMessages(postmanCollection.getItem(), httpMessages); + + requestor.run(httpMessages); } public PostmanCollection parse(String collectionJson) throws JsonProcessingException { @@ -83,7 +107,223 @@ public PostmanCollection parse(String collectionJson) throws JsonProcessingExcep return objectMapper.readValue(collectionJson, PostmanCollection.class); } + public static void extractHttpMessages( + List items, List httpMessages) { + for (AbstractItem item : items) { + if (item instanceof Item) { + HttpMessage httpMessage = extractHttpMessage((Item) item); + if (httpMessage != null) { + httpMessages.add(httpMessage); + } + } else if (item instanceof ItemGroup) { + extractHttpMessages(((ItemGroup) item).getItem(), httpMessages); + } + } + } + private static boolean isSupportedScheme(String scheme) { return "http".equalsIgnoreCase(scheme) || "https".equalsIgnoreCase(scheme); } + + private static boolean isContentTypeAlreadySet(List headers) { + if (headers != null) { + for (KeyValueData header : headers) { + if (header.getKey().equalsIgnoreCase(HttpHeader.CONTENT_TYPE)) { + return true; + } + } + } + return false; + } + + public static HttpMessage extractHttpMessage(Item item) { + Request request = item.getRequest(); + if (request == null) { + return null; + } + + Url url = request.getUrl(); + if (url == null) { + return null; + } + + HttpMessage httpMessage; + try { + String rawUrl = url.getRaw(); + httpMessage = new HttpMessage(new URI(rawUrl, false)); + } catch (URIException | HttpMalformedHeaderException | NullPointerException e) { + return null; + } + + httpMessage.getRequestHeader().setMethod(request.getMethod()); + + List headers = request.getHeader(); + if (headers != null) { + for (KeyValueData header : request.getHeader()) { + if (!header.isDisabled()) { + httpMessage.getRequestHeader().setHeader(header.getKey(), header.getValue()); + } + } + } + + Body body = request.getBody(); + if (body == null || body.isDisabled()) { + return httpMessage; + } + + String mode = body.getMode(); + if (mode == null) { + return httpMessage; + } + + String bodyContent = ""; + String contentType = ""; + + if (mode.equals(Body.RAW)) { + Map contentTypeMap = + Map.of( + "html", "text/html", + "javascript", "application/javascript", + "json", "application/json", + "xml", "application/xml"); + + contentType = "text/plain"; + + if (body.getOptions() != null && body.getOptions().getRaw() != null) { + String language = body.getOptions().getRaw().getLanguage(); + + if (language != null) { + contentType = + contentTypeMap.getOrDefault( + language.toLowerCase(Locale.ROOT), "text/html"); + } + } + + bodyContent = body.getRaw(); + } else if (mode.equals(Body.URL_ENCODED)) { + contentType = HttpHeader.FORM_URLENCODED_CONTENT_TYPE; + + StringBuilder urlencodedBodySB = new StringBuilder(); + + for (KeyValueData data : body.getUrlencoded()) { + if (!data.isDisabled()) { + if (urlencodedBodySB.length() > 0) { + urlencodedBodySB.append('&'); + } + try { + urlencodedBodySB + .append( + URLEncoder.encode( + data.getKey(), StandardCharsets.UTF_8.name())) + .append('=') + .append( + URLEncoder.encode( + data.getValue(), StandardCharsets.UTF_8.name())); + } catch (UnsupportedEncodingException e) { + } + } + } + + bodyContent = urlencodedBodySB.toString(); + } else if (mode.equals(Body.FORM_DATA)) { + String boundary = "----" + System.currentTimeMillis(); + + contentType = "multipart/form-data; boundary=" + boundary; + + StringBuilder formDataBodySB = new StringBuilder(); + for (FormData formData : body.getFormData()) { + if (!formData.isDisabled()) { + formDataBodySB + .append(generateMultiPartBody(formData, boundary)) + .append(HttpHeader.CRLF); + } + } + + formDataBodySB.append("--").append(boundary).append("--").append(HttpHeader.CRLF); + + bodyContent = formDataBodySB.toString(); + } else if (mode.equals(Body.FILE)) { + String src = body.getFile().getSrc(); + + contentType = getFileContentType(src); + + try { + bodyContent = FileUtils.readFileToString(new File(src), StandardCharsets.UTF_8); + } catch (IOException e1) { + } + } else if (mode.equals(Body.GRAPHQL)) { + contentType = HttpHeader.JSON_CONTENT_TYPE; + + GraphQl graphQlBody = body.getGraphQl(); + String query = graphQlBody.getQuery().replaceAll("\r\n", "\\\\r\\\\n"); + String variables = graphQlBody.getVariables().replaceAll("\\s", ""); + + bodyContent = String.format("{\"query\":\"%s\", \"variables\":%s}", query, variables); + } + + if (!isContentTypeAlreadySet(request.getHeader())) { + httpMessage.getRequestHeader().setHeader(HttpHeader.CONTENT_TYPE, contentType); + } + + httpMessage.getRequestBody().setBody(bodyContent.toString()); + httpMessage.getRequestHeader().setContentLength(httpMessage.getRequestBody().length()); + + return httpMessage; + } + + private static String generateMultiPartBody(FormData formData, String boundary) { + StringBuilder multipartDataSB = new StringBuilder(); + + multipartDataSB.append("--").append(boundary).append(HttpHeader.CRLF); + multipartDataSB + .append("Content-Disposition: form-data; name=\"") + .append(formData.getKey()) + .append("\""); + + if ("file".equals(formData.getType())) { + File file = new File(formData.getSrc()); + if (!file.exists() || !file.canRead() || !file.isFile()) { + return ""; + } + + multipartDataSB + .append("; filename=\"") + .append(file.getName()) + .append("\"") + .append(HttpHeader.CRLF); + + String propertyContentType = getFileContentType(formData.getSrc()); + if (!propertyContentType.isEmpty()) { + multipartDataSB + .append(HttpHeader.CONTENT_TYPE + ": ") + .append(propertyContentType) + .append(HttpHeader.CRLF); + } + + multipartDataSB.append(HttpHeader.CRLF); + + try { + String defn = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + multipartDataSB.append(defn); + } catch (IOException e) { + return ""; + } + } else { + multipartDataSB + .append(HttpHeader.CRLF) + .append(HttpHeader.CRLF) + .append(formData.getValue()); + } + + return multipartDataSB.toString(); + } + + private static String getFileContentType(String value) { + try { + String osAppropriatePath = value.startsWith("/") ? value.substring(1) : value; + return Files.probeContentType(Paths.get(osAppropriatePath)); + } catch (IOException e) { + return ""; + } + } } diff --git a/addOns/postman/src/main/java/org/zaproxy/addon/postman/Requestor.java b/addOns/postman/src/main/java/org/zaproxy/addon/postman/Requestor.java index 4e8736ab2ab..1ff2db848a7 100644 --- a/addOns/postman/src/main/java/org/zaproxy/addon/postman/Requestor.java +++ b/addOns/postman/src/main/java/org/zaproxy/addon/postman/Requestor.java @@ -20,11 +20,16 @@ package org.zaproxy.addon.postman; import java.io.IOException; +import java.util.List; import org.apache.commons.httpclient.URI; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.parosproxy.paros.network.HttpMessage; import org.parosproxy.paros.network.HttpSender; public class Requestor { + private static final Logger LOGGER = LogManager.getLogger(Requestor.class); + private int initiator; private HistoryPersister listener; private HttpSender sender; @@ -44,4 +49,15 @@ public String getResponseBody(URI uri) throws IOException { return httpRequest.getResponseBody().toString(); } + + public void run(List httpMessages) { + for (HttpMessage httpMessage : httpMessages) { + try { + sender.sendAndReceive(httpMessage, true); + listener.handleMessage(httpMessage, initiator); + } catch (IOException e) { + LOGGER.debug(e.getMessage(), e); + } + } + } } diff --git a/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/Body.java b/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/Body.java index e7f199b1810..0bdcf9fb4ab 100644 --- a/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/Body.java +++ b/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/Body.java @@ -20,6 +20,7 @@ package org.zaproxy.addon.postman.models; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import java.util.List; import org.zaproxy.addon.postman.deserializers.ListDeserializer; @@ -29,8 +30,14 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class Body { + public static final String RAW = "raw"; + public static final String URL_ENCODED = "urlencoded"; + public static final String FORM_DATA = "formdata"; + public static final String FILE = "file"; + public static final String GRAPHQL = "graphql"; + private static final List ALLOWED_MODES = - List.of("raw", "urlencoded", "formdata", "file", "graphql"); + List.of(RAW, URL_ENCODED, FORM_DATA, FILE, GRAPHQL); @JsonDeserialize(using = ObjectDeserializer.class) private String mode; @@ -42,12 +49,14 @@ public class Body { private List urlencoded; @JsonDeserialize(using = ListDeserializer.class) + @JsonProperty("formdata") private List formData; @JsonDeserialize(using = ObjectDeserializer.class) private File file; @JsonDeserialize(using = ObjectDeserializer.class) + @JsonProperty("graphql") private GraphQl graphQl; @JsonDeserialize(using = ObjectDeserializer.class) @@ -56,6 +65,12 @@ public class Body { @JsonDeserialize(using = ObjectDeserializer.class) private boolean disabled; + public Body() {} + + public Body(String mode) { + this.mode = mode; + } + public String getMode() { return mode; } @@ -144,6 +159,13 @@ public static class FormData extends KeyValueData { @JsonDeserialize(using = ObjectDeserializer.class) private String type; + public FormData() {} + + public FormData(String key, String value, String type) { + super(key, value); + this.type = type; + } + public String getSrc() { return src; } diff --git a/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/Item.java b/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/Item.java index 375ab177d6a..94ee70dbe9e 100644 --- a/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/Item.java +++ b/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/Item.java @@ -34,6 +34,12 @@ public class Item extends AbstractItem { @JsonDeserialize(using = ObjectDeserializer.class) private Request request; + public Item() {} + + public Item(Request request) { + this.request = request; + } + public Request getRequest() { return request; } diff --git a/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/ItemGroup.java b/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/ItemGroup.java index 91ba1d292cf..821c6fa76b7 100644 --- a/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/ItemGroup.java +++ b/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/ItemGroup.java @@ -35,6 +35,12 @@ public class ItemGroup extends AbstractItem { @JsonDeserialize(using = ListDeserializer.class) private List item; + public ItemGroup() {} + + public ItemGroup(List item) { + this.item = item; + } + public List getItem() { return item; } diff --git a/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/KeyValueData.java b/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/KeyValueData.java index be1f6d614d7..a2873bd7648 100644 --- a/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/KeyValueData.java +++ b/addOns/postman/src/main/java/org/zaproxy/addon/postman/models/KeyValueData.java @@ -20,11 +20,23 @@ package org.zaproxy.addon.postman.models; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import org.zaproxy.addon.postman.deserializers.ObjectDeserializer; +import org.zaproxy.addon.postman.models.Body.FormData; @JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) +@JsonSubTypes({@JsonSubTypes.Type(FormData.class)}) public class KeyValueData extends AbstractListElement { + public KeyValueData() {} + + public KeyValueData(String key, String value) { + this.key = key; + this.value = value; + } + @JsonDeserialize(using = ObjectDeserializer.class) private String key; @@ -32,7 +44,7 @@ public class KeyValueData extends AbstractListElement { private String value; @JsonDeserialize(using = ObjectDeserializer.class) - private Boolean disabled; + private boolean disabled; public String getKey() { return key; diff --git a/addOns/postman/src/test/java/org/zaproxy/addon/postman/PostmanParserUnitTest.java b/addOns/postman/src/test/java/org/zaproxy/addon/postman/PostmanParserUnitTest.java index 11ebf6cc171..1d24070a5f0 100644 --- a/addOns/postman/src/test/java/org/zaproxy/addon/postman/PostmanParserUnitTest.java +++ b/addOns/postman/src/test/java/org/zaproxy/addon/postman/PostmanParserUnitTest.java @@ -20,14 +20,37 @@ package org.zaproxy.addon.postman; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.params.provider.Arguments.arguments; import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.parosproxy.paros.network.HttpHeader; +import org.parosproxy.paros.network.HttpMessage; +import org.parosproxy.paros.network.HttpRequestHeader; +import org.zaproxy.addon.postman.models.AbstractItem; +import org.zaproxy.addon.postman.models.Body; +import org.zaproxy.addon.postman.models.Body.FormData; +import org.zaproxy.addon.postman.models.Body.GraphQl; +import org.zaproxy.addon.postman.models.Item; +import org.zaproxy.addon.postman.models.ItemGroup; +import org.zaproxy.addon.postman.models.KeyValueData; import org.zaproxy.addon.postman.models.PostmanCollection; +import org.zaproxy.addon.postman.models.Request; import org.zaproxy.zap.testutils.TestUtils; class PostmanParserUnitTest extends TestUtils { @@ -43,6 +66,93 @@ void teardown() throws Exception { stopServer(); } + static Stream extractionTestData() { + Item item = new Item(new Request("https://example.com")); + return Stream.of( + arguments(new ArrayList(List.of()), 0), + arguments(new ArrayList(List.of(item, item)), 2), + arguments( + new ArrayList( + List.of( + new ItemGroup( + new ArrayList(List.of(item, item))))), + 2), + arguments( + new ArrayList( + List.of( + item, + new ItemGroup(new ArrayList(List.of(item))))), + 2)); + } + + static Stream requestBodyTestData() throws URISyntaxException { + Body rawBody = new Body(Body.RAW); + rawBody.setRaw("raw-body"); + + Body urlencodedBody = new Body(Body.URL_ENCODED); + urlencodedBody.setUrlencoded( + new ArrayList<>( + List.of( + new KeyValueData("key1", "value1"), + new KeyValueData("key2", "value2")))); + + Body formDataBody = new Body(Body.FORM_DATA); + formDataBody.setFormData( + new ArrayList( + List.of( + new FormData("key1", "value1", "text"), + new FormData("key2", "", "file")))); + + GraphQl graphQl = new GraphQl(); + graphQl.setQuery( + "query getByArtist ($name: String!) {\r\n queryArtists (byName: $name) {\r\n name\r\n image\r\n albums {\r\n name\r\n }\r\n }\r\n}"); + graphQl.setVariables("{\r\n \"name\": \"{{artist}}\"\r\n}"); + + Body graphQlBody = new Body(Body.GRAPHQL); + graphQlBody.setGraphQl(graphQl); + + Body fileBody = new Body(Body.FILE); + fileBody.setFile(new org.zaproxy.addon.postman.models.Body.File()); + + return Stream.of( + arguments(rawBody, "text/plain", "raw-body"), + arguments( + urlencodedBody, + HttpRequestHeader.FORM_URLENCODED_CONTENT_TYPE, + "key1=value1&key2=value2"), + arguments( + formDataBody, + "multipart/form-data; boundary=----BOUNDARY", + "------BOUNDARY" + + "\r\n" + + "Content-Disposition: form-data; name=\"key1\"" + + "\r\n\r\n" + + "value1" + + "\r\n" + + "------BOUNDARY" + + "\r\n" + + "Content-Disposition: form-data; name=\"key2\"; filename=\"sampleFile.txt\"" + + "\r\n" + + "content-type: text/plain" + + "\r\n\r\n" + + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus eu tortor efficitur" + + "\r\n\r\n" + + "------BOUNDARY--" + + "\r\n"), + arguments( + graphQlBody, + HttpHeader.JSON_CONTENT_TYPE, + "{" + + "\"query\":\"query getByArtist ($name: String!) {\r\n queryArtists (byName: $name) {\r\n name\r\n image\r\n albums {\r\n name\r\n }\r\n }\r\n}\", " + .replaceAll("\r\n", "\\\\r\\\\n") + + "\"variables\":{\"name\":\"{{artist}}\"}".replaceAll("\\s", "") + + "}"), + arguments( + fileBody, + "text/plain", + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus eu tortor efficitur\r\n")); + } + @Test void shouldFailWhenCollectionIsInvalidJson() throws Exception { PostmanParser parser = new PostmanParser(); @@ -71,4 +181,70 @@ void shouldIgnoreUnKnownAttributes() throws Exception { String collectionJson = "{\"unKnown1\":true,\"unKnown2\":\"\"}"; assertDoesNotThrow(() -> parser.parse(collectionJson)); } + + @ParameterizedTest + @MethodSource("extractionTestData") + void shouldExtractHttpMessagesFromItems(List items, int numberOfitems) + throws Exception { + List httpMessages = new ArrayList<>(); + PostmanParser.extractHttpMessages(items, httpMessages); + + assertEquals(numberOfitems, httpMessages.size()); + } + + // The 'Content-Type' header gets set according to the mode of the request body, but if it's + // explicitly defined in the request, that value will take precedence + void shouldNotSetContentTypeIfExplicitlySet() { + Body body = new Body(); + body.setMode(Body.GRAPHQL); + + Request req = new Request("https://example.com"); + req.setBody(body); + req.setHeader( + Collections.singletonList( + new KeyValueData(HttpRequestHeader.CONTENT_TYPE, "custom-content-type"))); + + HttpMessage httpMessage = PostmanParser.extractHttpMessage(new Item(req)); + String contentType = + httpMessage.getRequestHeader().getHeader(HttpRequestHeader.CONTENT_TYPE); + + assertEquals("custom-content-type", contentType); + } + + @ParameterizedTest + @MethodSource("requestBodyTestData") + void shouldHandleRequestBodyModes(Body body, String contentType, String stringBody) { + Request req = new Request("https://example.com"); + req.setBody(body); + + if (contentType.startsWith("multipart/form-data")) { + Path path = getResourcePath("sampleFile.txt"); + String src = path.toAbsolutePath().toString(); + body.getFormData().get(1).setSrc(src); + } + + if (contentType.equals("text/plain") && stringBody.startsWith("Lorem")) { + Path path = getResourcePath("sampleFile.txt"); + String src = path.toAbsolutePath().toString(); + body.getFile().setSrc(src); + } + + HttpMessage httpMessage = PostmanParser.extractHttpMessage(new Item(req)); + + if (contentType.startsWith("multipart/form-data")) { + String tempStringBody = + new String(httpMessage.getRequestBody().getContent(), StandardCharsets.UTF_8); + String bodyFirstLine = tempStringBody.split("\r\n", 2)[0]; + String boundary = bodyFirstLine.split("------", 2)[1]; + contentType = contentType.replace("BOUNDARY", boundary); + stringBody = stringBody.replace("BOUNDARY", boundary); + } + + assertEquals( + contentType, + httpMessage.getRequestHeader().getHeader(HttpRequestHeader.CONTENT_TYPE)); + assertEquals( + stringBody, + new String(httpMessage.getRequestBody().getContent(), StandardCharsets.UTF_8)); + } } diff --git a/addOns/postman/src/test/resources/org/zaproxy/addon/postman/sampleFile.txt b/addOns/postman/src/test/resources/org/zaproxy/addon/postman/sampleFile.txt new file mode 100644 index 00000000000..37aee4a6bf4 --- /dev/null +++ b/addOns/postman/src/test/resources/org/zaproxy/addon/postman/sampleFile.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus eu tortor efficitur