/*
* Copyright 2014 GoDataDriven B.V.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.divolte.server;
import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
import javax.annotation.ParametersAreNonnullByDefault;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import io.divolte.server.config.ValidatedConfiguration;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.util.Utf8;
import org.junit.After;
import org.junit.Test;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.io.Resources;
import com.maxmind.geoip2.model.CityResponse;
import io.divolte.server.ServerTestUtils.EventPayload;
import io.divolte.server.ServerTestUtils.TestServer;
import io.divolte.server.ip2geo.LookupService;
import io.divolte.server.ip2geo.LookupService.ClosedServiceException;
import io.divolte.server.recordmapping.DslRecordMapper;
import io.divolte.server.recordmapping.SchemaMappingException;
@ParametersAreNonnullByDefault
public class DslRecordMapperTest {
private static final String CLIENT_SIDE_TIME = "i0rjfnxd";
private static final String DIVOLTE_URL_STRING = "http://localhost:%d/csc-event";
private static final String DIVOLTE_URL_QUERY_STRING = "?"
+ "p=0%3Ai0rjfnxc%3AJLOvH9Nda2c1uV8M~vmdhPGFEC3WxVNq&"
+ "s=0%3Ai0rjfnxc%3AFPpXFMdcEORvvaP_HbpDgABG3Iu5__4d&"
+ "v=0%3AOxVC1WJ4PZNEGIUuzdXPsy_bztnKMuoH&"
+ "e=0%3AOxVC1WJ4PZNEGIUuzdXPsy_bztnKMuoH0&"
+ "c=" + CLIENT_SIDE_TIME + "&"
+ "n=t&"
+ "f=t&"
+ "i=sg&"
+ "j=sg&"
+ "k=2&"
+ "w=sa&"
+ "h=sa&"
+ "t=pageView";
private static final String HOMOGENOUS_EVENT_PARAMS =
"u=" + encodeUrl("(sfoo!string!dbar!16!)");
private static final String HETEROGENOUS_EVENT_PARAMS =
"u=" + encodeUrl("(sfoo!string!dbar!16!aitems!" +
"(sname!apple!dcount!3!jprice!1.23!sextra1!ignored!)" +
"(sname!pear!dcount!1!jprice!0.89!sextra2!ignored!)" +
".)");
private static final String USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36";
public TestServer server;
private File mappingFile;
private File avroFile;
@Test
public void shouldPopulateFlatFields() throws InterruptedException, IOException {
setupServer("flat-mapping.groovy");
final EventPayload payload = request("https://example.com/", "http://example.com/");
final GenericRecord record = payload.record;
final DivolteEvent event = payload.event;
assertEquals(true, record.get("sessionStart"));
assertEquals(true, record.get("unreliable"));
assertEquals(false, record.get("dupe"));
assertEquals(event.requestStartTime.toEpochMilli(), record.get("ts"));
assertEquals("https://example.com/", record.get("location"));
assertEquals("http://example.com/", record.get("referer"));
assertEquals(USER_AGENT, record.get("userAgentString"));
assertEquals("Chrome", record.get("userAgentName"));
assertEquals("Chrome", record.get("userAgentFamily"));
assertEquals("Google Inc.", record.get("userAgentVendor"));
assertEquals("Browser", record.get("userAgentType"));
assertEquals("38.0.2125.122", record.get("userAgentVersion"));
assertEquals("Personal computer", record.get("userAgentDeviceCategory"));
assertEquals("OS X", record.get("userAgentOsFamily"));
assertEquals("10.10.1", record.get("userAgentOsVersion"));
assertEquals("Apple Computer, Inc.", record.get("userAgentOsVendor"));
assertEquals(event.partyId.value, record.get("client"));
assertEquals(event.sessionId.value, record.get("session"));
assertEquals(event.browserEventData.get().pageViewId, record.get("pageview"));
assertEquals(event.eventId, record.get("event"));
assertEquals(1018, record.get("viewportWidth"));
assertEquals(1018, record.get("viewportHeight"));
assertEquals(1024, record.get("screenWidth"));
assertEquals(1024, record.get("screenHeight"));
assertEquals(2, record.get("pixelRatio"));
assertEquals("pageView", record.get("eventType"));
Stream.of(
"sessionStart",
"unreliable",
"dupe",
"ts",
"location",
"referer",
"userAgentString",
"userAgentName",
"userAgentFamily",
"userAgentVendor",
"userAgentType",
"userAgentVersion",
"userAgentDeviceCategory",
"userAgentOsFamily",
"userAgentOsVersion",
"userAgentOsVendor",
"client",
"session",
"pageview",
"event",
"viewportWidth",
"viewportHeight",
"screenWidth",
"screenHeight",
"pixelRatio",
"eventType")
.forEach((v) -> assertNotNull(record.get(v)));
}
@Test
public void shouldMapClientTimestamp() throws IOException, InterruptedException {
setupServer("client-timestamp.groovy");
final EventPayload event = request("https://example.com/", "http://example.com/");
final GenericRecord record = event.record;
assertEquals(ClientSideCookieEventHandler.tryParseBase36Long(CLIENT_SIDE_TIME), record.get("ts"));
}
@Test(expected=SchemaMappingException.class)
public void shouldFailOnStartupIfMappingMissingField() throws IOException {
setupServer("missing-field-mapping.groovy");
}
@Test
public void shouldSetCustomCookieValue() throws InterruptedException, IOException {
setupServer("custom-cookie-mapping.groovy");
final EventPayload event = request("http://example.com");
assertEquals("custom_cookie_value", event.record.get("customCookie"));
}
@Test
public void shouldApplyActionsInClosureWhenEqualToConditionHolds() throws IOException, InterruptedException {
setupServer("when-mapping.groovy");
final EventPayload event = request("http://www.example.com/", "http://www.example.com/somepage.html");
assertEquals("locationmatch", event.record.get("eventType"));
assertEquals("referermatch", event.record.get("client"));
assertEquals(new Utf8("not set"), event.record.get("queryparam"));
assertEquals("absent", event.record.get("event"));
assertEquals("present", event.record.get("pageview"));
}
@Test
public void shouldChainValueProducersWithIntermediateNull() throws IOException, InterruptedException {
setupServer("chained-na-mapping.groovy");
final EventPayload event = request("http://www.exmaple.com/");
assertEquals(new Utf8("not set"), event.record.get("queryparam"));
}
@Test
public void shouldMatchRegexAndExtractGroups() throws IOException, InterruptedException {
setupServer("regex-mapping.groovy");
final EventPayload event = request("http://www.example.com/path/with/42/about.html", "http://www.example.com/path/with/13/contact.html");
assertEquals(true, event.record.get("pathBoolean"));
assertEquals("42", event.record.get("client"));
assertEquals("about", event.record.get("pageview"));
}
@Test
public void shouldParseUriComponents() throws IOException, InterruptedException {
setupServer("uri-mapping.groovy");
final EventPayload event = request(
"https://www.example.com:8080/path/to/resource/page.html?q=multiple+words+%24%23%25%26&p=10&p=20",
"http://example.com/path/to/resource/page.html?q=divolte&p=42#/client/side/path?x=value&y=42");
assertEquals("https", event.record.get("uriScheme"));
assertEquals("/path/to/resource/page.html", event.record.get("uriPath"));
assertEquals("www.example.com", event.record.get("uriHost"));
assertEquals(8080, event.record.get("uriPort"));
assertEquals("/client/side/path?x=value&y=42", event.record.get("uriFragment"));
assertEquals("q=multiple+words+$#%&&p=10&p=20", event.record.get("uriQueryString"));
assertEquals("multiple words $#%&", event.record.get("uriQueryStringValue"));
assertEquals(Arrays.asList("10", "20"), event.record.get("uriQueryStringValues"));
assertEquals(
ImmutableMap.of("p", Arrays.asList("10","20"), "q", Collections.singletonList("multiple words $#%&")),
event.record.get("uriQuery"));
}
@Test
public void shouldParseUriComponentsRaw() throws IOException, InterruptedException {
setupServer("uri-mapping-raw.groovy");
final EventPayload event = request(
"http://example.com/path/to/resource%20and%20such/page.html?q=multiple+words+%24%23%25%26&p=42#/client/side/path?x=value&y=42&q=multiple+words+%24%23%25%26");
assertEquals("/path/to/resource%20and%20such/page.html", event.record.get("uriPath"));
assertEquals("q=multiple+words+%24%23%25%26&p=42", event.record.get("uriQueryString"));
assertEquals("/client/side/path?x=value&y=42&q=multiple+words+%24%23%25%26", event.record.get("uriFragment"));
}
@Test
public void shouldParseMinimalUri() throws IOException, InterruptedException {
/*
* Test that URI parsing works on URIs that consist of only a path and possibly a query string.
* This is typical for Angular style applications, where the fragment component of the location
* is the internal location used within the angular app. In the mapping it should be possible
* to parse the fragment of the location to a URI again and do path matching and such against
* it.
*/
setupServer("uri-mapping-fragment.groovy");
final EventPayload event = request(
"http://example.com/path/?q=divolte#/client/side/path?x=value&y=42&q=multiple+words+%24%23%25%26");
assertEquals("/client/side/path", event.record.get("uriPath"));
assertEquals("x=value&y=42&q=multiple+words+%24%23%25%26", event.record.get("uriQueryString"));
assertEquals("multiple words $#%&", event.record.get("uriQueryStringValue"));
}
@Test
public void shouldNotFailOnBrokenQueryString() throws IOException, InterruptedException {
setupServer("funky-querystring-mapping.groovy");
final EventPayload event = request("http://example.com/path/?a=value&=42&b=&d=word&c&=bla");
/*
* Query string parsing semantics:
* ?q= => q == ""
* ?q=foo => q == "foo"
* ?q&a=bar => q == "" && a == "bar"
* ?=42&q=foo => q == "foo"
*/
assertEquals("value", event.record.get("uriQueryStringValue"));
assertEquals("", event.record.get("queryparam"));
assertEquals("", event.record.get("client"));
assertEquals("word", event.record.get("pageview"));
}
@Test
public void shouldSetCustomHeaders() throws IOException, InterruptedException {
setupServer("header-mapping.groovy");
final EventPayload event = request("http://www.example.com/");
assertEquals(Arrays.asList("first", "second", "last"), event.record.get("headerList"));
assertEquals("first", event.record.get("header"));
assertEquals("first,second,last", event.record.get("headers"));
}
@Test
public void shouldSetCustomEventParameters() throws IOException, InterruptedException {
setupServer("event-param-mapping.groovy");
final EventPayload event = request("http://www.example.com/", Collections.singletonList(HOMOGENOUS_EVENT_PARAMS));
assertEquals(ImmutableMap.of("foo", "string", "bar", "42"), event.record.get("paramMap"));
assertEquals("string", event.record.get("paramValue"));
}
@Test
public void shouldExtractJsonPathFromCustomEventParameters() throws IOException, InterruptedException {
setupServer("event-param-jsonpath-mapping.groovy");
final EventPayload event = request("http://www.example.com/", Collections.singletonList(HETEROGENOUS_EVENT_PARAMS));
assertEquals("string", event.record.get("paramValue"));
assertEquals(42, event.record.get("paramIntValue"));
assertEquals(Arrays.asList(1.23, 0.89), event.record.get("itemPrices"));
// Doing a proper check would require accessing the schema and building everything by hand.
// This is simpler and sufficient for the purposes of testing.
assertEquals("[{\"name\": \"apple\", \"count\": 3, \"price\": 1.23}, {\"name\": \"pear\", \"count\": 1, \"price\": 0.89}]",
GenericData.get().toString(event.record.get("items")));
}
@Test
public void shouldTreatEmptyJsonPathResultAsNonPresent() throws IOException, InterruptedException {
setupServer("event-param-jsonpath-missing.groovy");
final EventPayload event = request("http://www.example.com/");
assertEquals("value that should not be overwritten",
event.record.get("paramValue"));
}
@Test
public void shouldMapAllEventParameters() throws IOException, InterruptedException {
setupServer("event-param-all.groovy");
final EventPayload event = request("http://www.example.com/", Collections.singletonList(HETEROGENOUS_EVENT_PARAMS));
assertEquals("{\"foo\": \"string\", \"bar\": \"42\", \"items\": [{\"count\": 3, \"price\": 1.23}, {\"count\": 1, \"price\": 0.89}]}",
GenericData.get().toString(event.record.get("paramRecord")));
}
@Test
public void shouldTreatRuntimeEventParameterMappingMismatchAsNonPresent() throws IOException, InterruptedException {
setupServer("event-param-jsonpath-mismatch.groovy");
final EventPayload event = request("http://example.com/", Collections.singletonList(HETEROGENOUS_EVENT_PARAMS));
// Nothing should have been mapped here.
assertNull(event.record.get("paramIntValue"));
// This is mapped last: it should have completed even though an earlier field mapping failed.
assertTrue((Boolean)event.record.get("flag1"));
}
@Test
public void shouldSupportPresenceTestingOfJsonPathExpressions() throws IOException, InterruptedException {
setupServer("event-param-jsonpath-presence.groovy");
final EventPayload event = request("http://www.example.com/", Collections.singletonList(HOMOGENOUS_EVENT_PARAMS));
assertTrue((Boolean) event.record.get("flag1"));
assertFalse((Boolean) event.record.get("flag2"));
}
@Test
public void shouldMapAllGeoIpFields() throws IOException, InterruptedException, ClosedServiceException {
/*
* Have to work around not being able to create a HttpServerExchange a bit.
* We setup a actual server just to do a request and capture the HttpServerExchange
* instance. Then we setup a DslRecordMapper instance with a mock ip2geo lookup service,
* we then use the previously captured exchange object against our locally created mapper
* instance to test the ip2geo mapping (using the a mock lookup service).
*/
setupServer("minimal-mapping.groovy");
final EventPayload payload = request("http://www.example.com");
final File geoMappingFile = File.createTempFile("geo-mapping", ".groovy");
copyResourceToFile("geo-mapping.groovy", geoMappingFile);
final ImmutableMap<String, Object> mappingConfig = ImmutableMap.of(
"divolte.mappings.test.mapping_script_file", geoMappingFile.getAbsolutePath(),
"divolte.mappings.test.schema_file", avroFile.getAbsolutePath()
);
final Config geoConfig = ConfigFactory.parseMap(mappingConfig)
.withFallback(ConfigFactory.parseResources("base-test-server.conf"))
.withFallback(ConfigFactory.parseResources("reference-test.conf"));
final ValidatedConfiguration vc = new ValidatedConfiguration(() -> geoConfig);
final CityResponse mockResponseWithEverything = loadFromClassPath("/city-response-with-everything.json", new TypeReference<CityResponse>(){});
final Map<String,Object> expectedMapping = loadFromClassPath("/city-response-expected-mapping.json", new TypeReference<Map<String,Object>>(){});
final LookupService mockLookupService = mock(LookupService.class);
when(mockLookupService.lookup(any())).thenReturn(Optional.of(mockResponseWithEverything));
final DslRecordMapper mapper = new DslRecordMapper(
vc,
geoMappingFile.getAbsolutePath(),
new Schema.Parser().parse(Resources.toString(Resources.getResource("TestRecord.avsc"), StandardCharsets.UTF_8)),
Optional.of(mockLookupService));
final GenericRecord record = mapper.newRecordFromExchange(payload.event);
// Validate the results.
verify(mockLookupService).lookup(any());
verifyNoMoreInteractions(mockLookupService);
expectedMapping.forEach((k, v) -> {
final Object recordValue = record.get(k);
assertEquals("Property " + k + " not mapped correctly.", v, recordValue);
});
Files.delete(geoMappingFile.toPath());
}
@Test(expected=SchemaMappingException.class)
public void shouldFailOnIncompatibleTypesWithLiteral() throws IOException, InterruptedException {
setupServer("wrong-types-literal.groovy");
request("http://www.example.com/wrong");
}
@Test(expected=SchemaMappingException.class)
public void shouldFailOnIncompatibleTypesWithValueProducer() throws IOException, InterruptedException {
setupServer("wrong-types-producer.groovy");
request("http://www.example.com/wrong");
}
@Test
public void shouldMapLiteralsOntoCorrectTypes() throws IOException, InterruptedException {
setupServer("correct-types-literal.groovy");
final EventPayload event = request("http://www.example.com/correct");
assertEquals("string value", event.record.get("queryparam"));
assertEquals(true, event.record.get("queryparamBoolean"));
assertEquals(42L, event.record.get("queryparamLong"));
assertEquals(42, event.record.get("pathInteger"));
assertEquals(42.0, event.record.get("queryparamDouble"));
}
@Test
public void shouldStopWhenToldTo() throws IOException, InterruptedException {
setupServer("basic-stop.groovy");
final EventPayload event = request("http://www.example.com");
assertEquals("happened", event.record.get("client"));
assertNull(event.record.get("session"));
}
@Test
public void shouldStopOnNestedStop() throws IOException, InterruptedException {
setupServer("nested-conditional-stop.groovy");
final EventPayload event = request("http://www.example.com");
assertEquals("happened", event.record.get("client"));
assertNull(event.record.get("session"));
}
@Test
public void shouldStopOnCondition() throws IOException, InterruptedException {
setupServer("shorthand-conditional-stop.groovy");
final EventPayload event = request("http://www.example.com");
assertEquals("happened", event.record.get("client"));
assertNull(event.record.get("session"));
}
@Test
public void shouldStopOnConditionClosureSyntax() throws IOException, InterruptedException {
setupServer("shorthand-conditional-stop-closure.groovy");
final EventPayload event = request("http://www.example.com");
assertEquals("happened", event.record.get("client"));
assertNull(event.record.get("session"));
}
@Test
public void shouldStopOnTopLevelExit() throws IOException, InterruptedException {
setupServer("basic-toplevel-exit.groovy");
final EventPayload event = request("http://www.example.com");
assertEquals("happened", event.record.get("client"));
assertNull(event.record.get("session"));
}
@Test
public void shouldExitFromSectionOnCondition() throws IOException, InterruptedException {
setupServer("nested-conditional-exit.groovy");
final EventPayload event = request("http://www.example.com");
assertEquals("happened", event.record.get("client"));
assertEquals("happened", event.record.get("pageview"));
assertEquals("happened", event.record.get("event"));
assertEquals("happened", event.record.get("customCookie"));
assertNull(event.record.get("session"));
}
@Test
public void shouldExitFromSectionOnConditionClosureSyntax() throws IOException, InterruptedException {
setupServer("nested-conditional-exit-closure.groovy");
final EventPayload event = request("http://www.example.com");
assertEquals("happened", event.record.get("client"));
assertEquals("happened", event.record.get("pageview"));
assertEquals("happened", event.record.get("event"));
assertEquals("happened", event.record.get("customCookie"));
assertNull(event.record.get("session"));
}
@Test
public void shouldApplyBooleanLogic() throws IOException, InterruptedException {
setupServer("boolean-logic.groovy");
final EventPayload event = request("http://www.example.com/");
assertTrue((Boolean) event.record.get("unreliable"));
assertFalse((Boolean) event.record.get("dupe"));
assertTrue((Boolean) event.record.get("queryparamBoolean"));
assertTrue((Boolean) event.record.get("pathBoolean"));
}
private static final ObjectMapper MAPPER =
new ObjectMapper()
.configure(JsonParser.Feature.ALLOW_COMMENTS, true)
.setInjectableValues(new InjectableValues.Std().addValue("locales", ImmutableList.of("en")));
private <T> T loadFromClassPath(final String resource, final TypeReference<?> typeReference) throws IOException {
try (final InputStream resourceStream = this.getClass().getResourceAsStream(resource)) {
return MAPPER.readValue(resourceStream, typeReference);
}
}
private EventPayload request(final String location) throws IOException, InterruptedException {
return request(location, Collections.emptyList());
}
private EventPayload request(final String location, final String referer) throws IOException, InterruptedException {
return request(location, Collections.singletonList("r=" + encodeUrl(referer)));
}
private EventPayload request(final String location,
final List<String> extraEncodedQueryParameters) throws IOException, InterruptedException {
final StringBuilder urlBuilder = new StringBuilder(String.format(DIVOLTE_URL_STRING, server.port));
urlBuilder.append(DIVOLTE_URL_QUERY_STRING)
.append("&l=").append(encodeUrl(location));
extraEncodedQueryParameters.forEach(s -> urlBuilder.append('&').append(s));
final URL url = new URL(urlBuilder.toString());
final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.addRequestProperty("User-Agent", USER_AGENT);
conn.addRequestProperty("Cookie", "custom_cookie=custom_cookie_value;");
conn.addRequestProperty("X-Divolte-Test", "first");
conn.addRequestProperty("X-Divolte-Test", "second");
conn.addRequestProperty("X-Divolte-Test", "last");
conn.setRequestMethod("GET");
assertEquals(200, conn.getResponseCode());
return server.waitForEvent();
}
private static String encodeUrl(final String s) {
try {
return URLEncoder.encode(s, StandardCharsets.UTF_8.name());
} catch (final UnsupportedEncodingException e) {
// This should never happen: all platforms must support UTF-8.
throw new RuntimeException(e);
}
}
private void setupServer(final String mapping) throws IOException {
mappingFile = File.createTempFile("test-mapping", ".groovy");
copyResourceToFile(mapping, mappingFile);
avroFile = File.createTempFile("TestSchema-", ".avsc");
copyResourceToFile("TestRecord.avsc", avroFile);
final ImmutableMap<String, Object> mappingConfig = ImmutableMap.of(
"divolte.mappings.test.mapping_script_file", mappingFile.getAbsolutePath(),
"divolte.mappings.test.schema_file", avroFile.getAbsolutePath()
);
server = new TestServer("base-test-server.conf", mappingConfig);
}
private static void copyResourceToFile(final String resourceName, final File file) throws IOException {
com.google.common.io.Files.write(Resources.toByteArray(Resources.getResource(resourceName)), file);
}
@After
public void shutdown() throws IOException {
if (server != null) {
server.server.shutdown();
}
if (mappingFile != null) {
Files.delete(mappingFile.toPath());
}
if (avroFile != null) {
Files.delete(avroFile.toPath());
}
}
}