osmlab · matthieun · Nov 13, 2019 · Nov 12, 2019 · Nov 12, 2019 · Nov 13, 2019
diff --git a/README.md b/README.md
@@ -47,6 +47,7 @@ As well as other handy tools:
 * [Higher-level entities](/src/main/java/org/openstreetmap/atlas/geography/atlas/items/complex#complex-entities)
 * [Saving](/src/main/java/org/openstreetmap/atlas/geography/atlas#saving-an-atlas) / [Loading](/src/main/java/org/openstreetmap/atlas/geography/atlas#using-atlas)
 * [Command Line Tools](atlas-shell-tools)
+* [Atlas Query Language i.e. AQL](/src/main/groovy/org/openstreetmap/atlas/geography/atlas/dsl#README.md)
 
 # Community
 

diff --git a/build.gradle b/build.gradle
@@ -1,5 +1,6 @@
 plugins {
 	id 'java'
+	id 'groovy'
 	id 'maven'
 	id 'maven-publish'
 	id 'idea'
@@ -66,6 +67,7 @@ dependencies
     compile packages.groovy
     compile packages.checkstyle
     compile packages.diff_utils
+    compile packages.groovy_json
 
     testCompile packages.checkstyle_tests
 

diff --git a/dependencies.gradle b/dependencies.gradle
@@ -30,6 +30,7 @@ project.ext.versions = [
     groovy: '2.5.4',
     atlas_checkstyle: '5.6.9',
     diff_utils: '4.0',
+    groovy_json: '2.5.4'
 ]
 
 project.ext.packages = [
@@ -77,4 +78,5 @@ project.ext.packages = [
     checkstyle_tests: "com.puppycrawl.tools:checkstyle:${versions.checkstyle}:tests",
     atlas_checkstyle: "org.openstreetmap.atlas:atlas:${versions.atlas_checkstyle}",
     diff_utils: "io.github.java-diff-utils:java-diff-utils:${versions.diff_utils}",
+    groovy_json: "org.codehaus.groovy:groovy-json:${versions.groovy_json}"
 ]
diff --git a/src/main/groovy/org/openstreetmap/atlas/geography/atlas/dsl/README.md b/src/main/groovy/org/openstreetmap/atlas/geography/atlas/dsl/README.md
@@ -0,0 +1,120 @@
+# Atlas Query Language (AQL)
+
+# Introduction
+Atlas Query Language or AQL is a new feature that allows developers to write queries directly against the Atlas files. This will allow rapid development where developers can write select statements against their Atlas or OSM files locally and then port these queries in their applications (spark or otherwise) to run the same queries at scale.
+
+AQL currently supports using (as in use Atlas/OSM files), select, update, delete, explain (as in explain plan), commit and diff (as in difference between two atlas schemas).
+
+The AQL statements (select, update, delete) support complex where clauses with geospatial and tag querying among other criterion. AQL also supports nested inner queries..
+
+AQL leverages Atlas as the underlying framework and doesn't reinvent the wheels. AQL is efficient in the sense that queries are automatically optimized to benefit from index-ing which can be either atlas id based or geospatial based.
+
+Atlas allows working against all 6 major entities, node, point, edge, line, relation, area, exposed as "tables" in an atlas schema.
+
+# Quick Start Guide
+
+The easiest way to get started is to have an IDE with Groovy support. Ensure that you have Atlas 
+as a dependency. IntelliJ with Groovy plugin offers excellent support including auto-complete of
+queries as demonstrated below.
+
+- Imports
+
+Create a groovy file with 2 imports listed below. Notice these are `static` star imports,
+
+```groovy
+/*Optional package statement if applicable.*/
+
+import static org.openstreetmap.atlas.geography.atlas.dsl.query.QueryBuilderFactory.*
+import static org.openstreetmap.atlas.geography.atlas.dsl.schema.AtlasDB.getEdge
+```
+
+- Load Atlas (or OSM) file(s)
+
+```sql
+/*
+examples,
+a. One Atlas File on disk:- file:///home/me/dir1/dir2//ButterflyPark.atlas
+b. One Atlas File omn disk (without explicitly mentioning file: scheme):- /home/me/dir1/dir2//ButterflyPark.atlas
+d. All Atlas files in a directory:- /home/dir1/dir2/allAtlasesInsideDirNoRecurse
+e. A file from the classpath:- classpath:/data/ButterflyPark/ButterflyPark.osm
+f. Multiple files from classpath:- classpath:/atlas1/something1.atlas,something2.atlas;/atlas2/Alcatraz.atlas,Butterfly.atlas
+*/
+myAtlas = using "<your atlas or OSM file>"
+```
+
+- Write queries against `myAtlas`
+
+Here are some examples,
+
+* Select everything from node "table"
+
+```sql
+select node._ from myAtlas.node
+```
+
+* Select with where clause
+
+```sql
+select node.id, node.osmId, node.tags from myAtlas.node where node.hasId(123000000) or node.hasTag("amenity": "college")
+```
+
+* Update statement
+
+```sql
+update myAtlas.node set node.addTag(hello: "world") where node.hasIds(123456789, 9087655432)
+```
+
+* Delete statement
+
+```sql
+delete myAtlas.edge where edge.hasTag(highway: "footway") and not(edge.hasTag(foot: "yes"))
+```
+
+* An example where we commit to create a new Atlas (schema),
+
+```sql
+
+/*Load*/ 
+atlas = using "classpath:/data/Alcatraz/Alcatraz.osm"
+
+/*Run some select queries to explore the data.*/
+select edge._ from atlas.edge limit 100
+
+/*
+Run some update and/or delete statement(s)
+*/
+update1 = update atlas.edge set edge.addTag(website: "https://www.nps.gov/alca") where edge.hasTagLike(name: /Pier/) or edge.hasTagLike(name: /Island/) or edge.hasTagLike(name: /Road/)
+update2 = update atlas.edge set edge.addTag(wikipedia: "https://en.wikipedia.org/wiki/Alcatraz_Island") where edge.hasTagLike(/foot/)
+
+/*
+Note edge.hasTagLike(name: /Pier/), in groovy you can use forward slashes instead of double 
+quotes to work with RegEx and that allows you to skip the double escaping that we do in Java 
+when working with RegExes.
+*/
+
+/*
+Commit the changes
+*/
+changedAtlas = commit update1, update2
+
+/*
+Run Selects against the new atlas schema, notice changedAtlas.edge instead of atlas.edge here,
+*/
+select edge._ from changedAtlas.edge where edge.hasTag("website") or edge.hasTag("wikipedia")
+
+/*
+Run some commands like diff and explain plan
+*/
+diff atlas, changedAtlas
+
+explain update1
+```
+
+# Supported Tables
+
+- node
+- point
+- line
+- edge
+- relation
+- area
diff --git a/...in/groovy/org/openstreetmap/atlas/geography/atlas/dsl/authentication/Authenticator.groovy b/...in/groovy/org/openstreetmap/atlas/geography/atlas/dsl/authentication/Authenticator.groovy
@@ -0,0 +1,13 @@
+package org.openstreetmap.atlas.geography.atlas.dsl.authentication
+
+/**
+ * A general purpose message authenticator. See <a href="https://en.wikipedia.org/wiki/Message_authentication">wiki</a>
+ * for more details on message authentication.
+ *
+ * @author Yazad Khambata
+ */
+interface Authenticator {
+    String sign(String message)
+
+    void verify(String message, String signature)
+}
diff --git a/...treetmap/atlas/geography/atlas/dsl/authentication/impl/SHA512HMACAuthenticatorImpl.groovy b/...treetmap/atlas/geography/atlas/dsl/authentication/impl/SHA512HMACAuthenticatorImpl.groovy
@@ -0,0 +1,52 @@
+package org.openstreetmap.atlas.geography.atlas.dsl.authentication.impl
+
+import org.apache.commons.lang3.Validate
+import org.openstreetmap.atlas.geography.atlas.dsl.authentication.Authenticator
+import org.openstreetmap.atlas.geography.atlas.dsl.util.Valid
+
+import javax.crypto.Mac
+import javax.crypto.spec.SecretKeySpec
+
+/**
+ * This implementation performs message authentication uses <a href="https://en.wikipedia.org/wiki/HMAC">HMAC</a>
+ * with <a href="https://en.wikipedia.org/wiki/SHA-2">SHA512</a>
+ * (read more about MACs on <a href="https://en.wikipedia.org/wiki/Message_authentication_code">wiki</a>).
+ *
+ * HMAC here is implemented utilizing Java Cryptography Architecture or JCA. Read Oracle documentation on JCA
+ * <a href="https://docs.oracle.com/javase/8/docs/technotes/guides/security/crypto/CryptoSpec.html">here</a>.
+ *
+ * @author Yazad Khambata
+ */
+class SHA512HMACAuthenticatorImpl implements Authenticator {
+
+    public static final int MIN_KEY_SIZE = 10
+    private String key
+
+    private static final String HMAC_SHA512 = "HmacSHA512"
+
+    SHA512HMACAuthenticatorImpl(final String key) {
+        super()
+        Valid.notEmpty key
+
+        Valid.isTrue(key.size() >= MIN_KEY_SIZE, "key should be at least ${MIN_KEY_SIZE}.")
+        this.key = key
+    }
+
+    @Override
+    String sign(final String message) {
+        final byte[] byteKey = key.getBytes("UTF-8")
+        final Mac sha512_HMAC = Mac.getInstance(HMAC_SHA512)
+        final SecretKeySpec keySpec = new SecretKeySpec(byteKey, HMAC_SHA512)
+        sha512_HMAC.init(keySpec)
+
+        final byte[] mac_data = sha512_HMAC.doFinal(message.getBytes("UTF-8"))
+        final String result = Base64.encoder.encodeToString(mac_data)
+
+        result
+    }
+
+    @Override
+    void verify(final String message, final String signature) {
+        Validate.isTrue(sign(message) == signature, "Signature Mismatch; message: [${message}]; signature: [${signature}].")
+    }
+}
diff --git a/src/main/groovy/org/openstreetmap/atlas/geography/atlas/dsl/console/ConsoleWriter.groovy b/src/main/groovy/org/openstreetmap/atlas/geography/atlas/dsl/console/ConsoleWriter.groovy
@@ -0,0 +1,14 @@
+package org.openstreetmap.atlas.geography.atlas.dsl.console
+
+/**
+ * Decouple using PrintStream or Logger in the statements and commends.
+ *
+ * @author Yazad Khambata
+ */
+interface ConsoleWriter {
+    boolean isTurnedOff()
+
+    void echo(String text)
+
+    void echo(Object object)
+}
diff --git a/.../groovy/org/openstreetmap/atlas/geography/atlas/dsl/console/impl/BaseConsoleWriter.groovy b/.../groovy/org/openstreetmap/atlas/geography/atlas/dsl/console/impl/BaseConsoleWriter.groovy
@@ -0,0 +1,41 @@
+package org.openstreetmap.atlas.geography.atlas.dsl.console.impl
+
+import org.openstreetmap.atlas.geography.atlas.dsl.console.ConsoleWriter
+import org.openstreetmap.atlas.geography.atlas.dsl.util.Valid
+
+import java.util.function.Consumer
+
+/**
+ * Base class of console writers.
+ *
+ * @author Yazad Khambata
+ */
+abstract class BaseConsoleWriter implements ConsoleWriter {
+    private boolean turnedOff
+
+    private Consumer<String> echoer
+
+    protected BaseConsoleWriter(final Consumer<String> echoer) {
+        this.echoer = echoer
+        this.turnedOff = (echoer == null)
+    }
+
+    @Override
+    boolean isTurnedOff() {
+        turnedOff
+    }
+
+    @Override
+    void echo(final String text) {
+        if (isTurnedOff()) {
+            return
+        }
+
+        echoer.accept(text)
+    }
+
+    @Override
+    void echo(final Object object) {
+        echo(((String)(object?.toString())))
+    }
+}
diff --git a/...groovy/org/openstreetmap/atlas/geography/atlas/dsl/console/impl/QuietConsoleWriter.groovy b/...groovy/org/openstreetmap/atlas/geography/atlas/dsl/console/impl/QuietConsoleWriter.groovy
@@ -0,0 +1,24 @@
+package org.openstreetmap.atlas.geography.atlas.dsl.console.impl
+
+/**
+ * Permanently turned off console writer.
+ *
+ * @author Yazad Khambata
+ */
+final class QuietConsoleWriter extends BaseConsoleWriter {
+
+    private static QuietConsoleWriter instance = new QuietConsoleWriter()
+
+    private QuietConsoleWriter() {
+        super(null)
+    }
+
+    @Override
+    void echo(final String text) {
+        //NOP
+    }
+
+    static QuietConsoleWriter getInstance() {
+        instance
+    }
+}
diff --git a/...g/openstreetmap/atlas/geography/atlas/dsl/console/impl/StandardOutputConsoleWriter.groovy b/...g/openstreetmap/atlas/geography/atlas/dsl/console/impl/StandardOutputConsoleWriter.groovy
@@ -0,0 +1,27 @@
+package org.openstreetmap.atlas.geography.atlas.dsl.console.impl
+
+import org.openstreetmap.atlas.geography.atlas.dsl.util.Valid
+
+/**
+ * Works against standard output.
+ *
+ * @author Yazad Khambata
+ */
+class StandardOutputConsoleWriter extends BaseConsoleWriter {
+
+    private static final StandardOutputConsoleWriter instance = new StandardOutputConsoleWriter(System.out)
+
+    private StandardOutputConsoleWriter(final PrintStream printStream) {
+        super(toEchoer(printStream))
+    }
+
+    private static Closure toEchoer(final PrintStream printStream) {
+        Valid.notEmpty printStream
+
+        { text -> printStream.println(text) }
+    }
+
+    static StandardOutputConsoleWriter getInstance() {
+        instance
+    }
+}
diff --git a/src/main/groovy/org/openstreetmap/atlas/geography/atlas/dsl/engine/QueryExecutor.groovy b/src/main/groovy/org/openstreetmap/atlas/geography/atlas/dsl/engine/QueryExecutor.groovy
@@ -0,0 +1,17 @@
+package org.openstreetmap.atlas.geography.atlas.dsl.engine
+
+import org.openstreetmap.atlas.geography.atlas.Atlas
+import org.openstreetmap.atlas.geography.atlas.dsl.query.result.Result
+
+/**
+ * Used as an integration point between other applications and processes.
+ * The query is expected as an input String or a Reader.
+ *
+ * @author Yazad Khambata
+ */
+interface QueryExecutor {
+
+    Result exec(Atlas atlas, String queryAsString, final String signature)
+
+    Result exec(Atlas atlas, Reader queryAsReader, final String signature)
+}